chellow 1726510224.0.0__py3-none-any.whl → 1726664509.0.0__py3-none-any.whl

chellow/e/bill_parsers/mm.py

@@ -217,7 +217,7 @@ def _handle_0860(headers, pre_record, record):
 
 def _handle_1455(headers, pre_record, record):
     parts = _chop_record(
-        record, ccl_kwh=13, unknown_1=8, ccl_rate=15, ccl_gbp=12, unkown_2=8
+        record, ccl_kwh=13, unknown_1=10, ccl_rate=13, ccl_gbp=12, unkown_2=8
     )
     bd = headers["breakdown"]
     bd["ccl-kwh"] += Decimal(parts["ccl_kwh"])

chellow/e/hh_importer.py

@@ -8,7 +8,7 @@ import traceback
 from collections import defaultdict, deque
 from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal
-from io import StringIO, TextIOWrapper
+from io import StringIO
 from pathlib import Path
 
 import jinja2
@@ -40,7 +40,7 @@ from chellow.utils import (
 processes = defaultdict(list)
 tasks = {}
 
-extensions = [".df2", ".simple.csv", ".bg.csv"]
+extensions = [".df2", ".simple.csv", ".bg.csv", ".vital.xlsx"]
 
 
 class HhDataImportProcess(threading.Thread):
@@ -277,7 +277,7 @@ class HhImportTask(threading.Thread):
             self.log(f"File size is {fsize} bytes.")
             self.log(f"Treating files as type {file_type}")
             self.importer = HhDataImportProcess(
-                self.contract_id, 0, TextIOWrapper(f, "utf8"), fpath + file_type, fsize
+                self.contract_id, 0, f, fpath + file_type, fsize
             )
 
             self.importer.run()
@@ -412,7 +412,7 @@ class HhImportTask(threading.Thread):
             self.log(f"File size is {fsize} bytes.")
             self.log(f"Treating files as type {file_type}")
             self.importer = HhDataImportProcess(
-                self.contract_id, 0, TextIOWrapper(f, "utf8"), fpath + file_type, fsize
+                self.contract_id, 0, f, fpath + file_type, fsize
             )
 
             self.importer.run()

chellow/e/hh_parser_df2.py

@@ -1,5 +1,6 @@
 import decimal
 import itertools
+from codecs import iterdecode
 
 from werkzeug.exceptions import BadRequest
 
@@ -13,7 +14,7 @@ def create_parser(reader, mpan_map, messages):
 class StarkDf2HhParser:
     def __init__(self, reader, mpan_map, messages):
         self.line = self.line_number = None
-        self.reader = zip(itertools.count(1), reader)
+        self.reader = zip(itertools.count(1), iterdecode(reader, "utf-8"))
 
         self.line_number, self.line = next(self.reader)
         if self.line.strip().upper() != "#F2":

chellow/e/hh_parser_simple_csv.py

@@ -1,5 +1,6 @@
 import csv
 import itertools
+from codecs import iterdecode
 from datetime import datetime as Datetime
 from decimal import Decimal
 
@@ -14,7 +15,8 @@ def create_parser(reader, mpan_map, messages):
 
 class HhParserCsvSimple:
     def __init__(self, reader, mpan_map, messages):
-        self.shredder = zip(itertools.count(1), csv.reader(reader))
+        s = iterdecode(reader, "utf-8")
+        self.shredder = zip(itertools.count(1), csv.reader(s))
         next(self.shredder)  # skip the title line
         self.values = None
 

chellow/e/hh_parser_vital_xlsx.py

@@ -0,0 +1,132 @@
+from datetime import datetime as Datetime
+
+from decimal import Decimal, InvalidOperation
+from openpyxl import load_workbook
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import HH, ct_datetime, hh_before, hh_range, to_utc
+
+
+def create_parser(input_stream, mpan_map, messages):
+    return HhParserVital(input_stream, mpan_map, messages)
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_date(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    val = cell.value
+    if not isinstance(val, Datetime):
+        raise BadRequest(
+            f"Problem reading {val} (of type {type(val)}) as a timestamp at "
+            f"{cell.coordinate}."
+        )
+    return val
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_dec(sheet, col, row):
+    return get_dec_from_cell(get_cell(sheet, col, row))
+
+
+def get_dec_from_cell(cell):
+    try:
+        return Decimal(str(cell.value))
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+def find_hhs(sheet, imp_mpan, exp_mpan):
+    row = pres_readings = pres_ts = None
+    caches = {}
+    try:
+        for row in range(2, len(sheet["A"]) + 1):
+            imp_read_cell = get_cell(sheet, "H", row)
+            imp_read_val = imp_read_cell.value
+            if imp_read_val is None or imp_read_val == "":
+                break
+
+            exp_read_cell = get_cell(sheet, "I", row)
+            exp_read_val = exp_read_cell.value
+            if exp_read_val is None or exp_read_val == "":
+                break
+
+            # '16/09/2024 14:45:00
+            ts_str = get_str(sheet, "A", row)
+            ts_naive = Datetime.strptime(ts_str, "%d/%m/%Y %H:%M:%S")
+            ts_ct = ct_datetime(
+                ts_naive.year,
+                ts_naive.month,
+                ts_naive.day,
+                ts_naive.hour,
+                ts_naive.minute,
+            )
+
+            imp_read = get_dec_from_cell(imp_read_cell)
+            exp_read = get_dec_from_cell(exp_read_cell)
+            ts = to_utc(ts_ct)
+
+            if ts_naive.minute in (0, 30) and hh_before(ts, pres_ts):
+                if pres_readings is not None:
+                    pres_imp_read, pres_exp_read = pres_readings
+                    hhs = hh_range(caches, ts, pres_ts - HH)
+                    num_hhs = len(hhs)
+                    val_imp = (pres_imp_read - imp_read) / num_hhs
+                    val_exp = (pres_exp_read - exp_read) / num_hhs
+                    for hh in hhs:
+
+                        yield {
+                            "mpan_core": imp_mpan,
+                            "channel_type": "ACTIVE",
+                            "start_date": hh,
+                            "value": val_imp,
+                            "status": "A",
+                        }
+                        yield {
+                            "mpan_core": exp_mpan,
+                            "channel_type": "ACTIVE",
+                            "start_date": hh,
+                            "value": val_exp,
+                            "status": "A",
+                        }
+
+                pres_readings = imp_read, exp_read
+                pres_ts = ts
+
+    except BadRequest as e:
+        e.description = (f"Problem at row: {row}: {e.description}",)
+        raise e
+
+
+class HhParserVital:
+    def __init__(self, input_stream, mpan_map, messages):
+        book = load_workbook(input_stream, data_only=True)
+        sheet = book.worksheets[0]
+        imp_name = get_str(sheet, "H", 1).strip()
+        imp_mpan = mpan_map[imp_name]
+        exp_name = get_str(sheet, "I", 1).strip()
+        exp_mpan = mpan_map[exp_name]
+        self.data = iter(find_hhs(sheet, imp_mpan, exp_mpan))
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return next(self.data)
+
+    def close(self):
+        pass

chellow/e/views.py

@@ -3,7 +3,7 @@ import os
 import threading
 from collections import defaultdict
 from datetime import datetime as Datetime
-from io import StringIO
+from io import BytesIO, StringIO
 from itertools import chain, islice
 from random import random
 
@@ -1233,7 +1233,7 @@ def dc_contracts_hh_imports_post(contract_id):
         contract = Contract.get_dc_by_id(g.sess, contract_id)
 
         file_item = req_file("import_file")
-        f = StringIO(str(file_item.stream.read(), "utf-8"))
+        f = BytesIO(file_item.stream.read())
         f.seek(0, os.SEEK_END)
         file_size = f.tell()
         f.seek(0)

chellow/gas/bill_parser_total_edi.py

@@ -110,10 +110,10 @@ def _process_CCD2(elements, headers):
     units_billed = to_decimal(nuct) / units_divisor
     breakdown[key] += units_billed
 
-    if "start_date" not in headers:
+    if tpref == "standing" or "start_date" not in headers:
         headers["start_date"] = to_date(csdt[0])
 
-    if "finish_date" not in headers:
+    if tpref == "standing" or "finish_date" not in headers:
         headers["finish_date"] = _to_finish_date(cedt[0])
 
     if "mprn" not in headers:

chellow/reports/report_111.py

@@ -2,7 +2,7 @@ import csv
 import sys
 import threading
 import traceback
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
 from datetime import datetime as Datetime
 from decimal import Decimal
 from itertools import combinations
@@ -11,7 +11,7 @@ from dateutil.relativedelta import relativedelta
 
 from flask import g, request
 
-from sqlalchemy import or_
+from sqlalchemy import or_, select
 from sqlalchemy.orm import joinedload, subqueryload
 from sqlalchemy.sql.expression import null, true
 
@@ -27,7 +27,6 @@ from chellow.models import (
     Contract,
     Era,
     Llfc,
-    MarketRole,
     MtcParticipant,
     RSession,
     RegisterRead,
@@ -380,19 +379,19 @@ def _process_supply(
         while enlarged:
             enlarged = False
             covered_elems = find_elements(bill)
-            covered_bills = OrderedDict(
+            covered_bills = dict(
                 (b.id, b)
-                for b in sess.query(Bill)
-                .join(Batch)
-                .join(Contract)
-                .join(MarketRole)
-                .filter(
-                    Bill.supply == supply,
-                    Bill.start_date <= covered_finish,
-                    Bill.finish_date >= covered_start,
-                    MarketRole.code == market_role_code,
+                for b in sess.scalars(
+                    select(Bill)
+                    .join(Batch)
+                    .where(
+                        Bill.supply == supply,
+                        Bill.start_date <= covered_finish,
+                        Bill.finish_date >= covered_start,
+                        Batch.contract == contract,
+                    )
+                    .order_by(Bill.start_date, Bill.issue_date)
                 )
-                .order_by(Bill.start_date, Bill.issue_date)
             )
             while True:
                 to_del = None

chellow/templates/e/dc_contract_hh_import.html

@@ -21,15 +21,16 @@
 				Reached line number {{process.converter.line_number}}.
 			{% endif %}
 		</p>
-	{% elif process.messages|length == 0 %}
+	{% elif process.error is none %}
 		<p>The import has completed successfully.</p>
 	{% else %}
-		<p>The import failed:</p>
-		<ul>
-			{% for message in process.messages %}
-				<li>{{message}}</li>
-			{% endfor %}
-		</ul>
+		<p>The import failed. {{process.error}}</p>
 	{% endif %}
+	
+	<ul>
+		{% for message in process.messages %}
+			<li>{{message}}</li>
+		{% endfor %}
+	</ul>
 
 {% endblock %}

{chellow-1726510224.0.0.dist-info → chellow-1726664509.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1726510224.0.0
+Version: 1726664509.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1726510224.0.0.dist-info → chellow-1726664509.0.0.dist-info}/RECORD

@@ -24,10 +24,11 @@ chellow/e/dno_rate_parser.py,sha256=A5TP6KjyfT5lVWh7dX4SiXRi6wnf2lGv-H_T4Sod8CI,
 chellow/e/duos.py,sha256=nwviRjz-qIt3GxIMHk0hItIT4dtKsxOWq9TUC1z-hO8,30864
 chellow/e/elexon.py,sha256=ALhXS9Es7PV0z9ukPbIramn3cf3iLyFi-PMWPSm5iOs,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
-chellow/e/hh_importer.py,sha256=-jvJaex756NpygSapsWxXl6RK0Z4ymkz3QA6ETrdnac,21476
+chellow/e/hh_importer.py,sha256=vD2PRMSBhxmsyJ4xEq2N7Ib5Ohmf1FmQk8-APw8vBEs,21430
 chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
-chellow/e/hh_parser_df2.py,sha256=7gaVidyIqeqprm3zYPtHvRLLn08nph04N0cKlveNTV0,4279
-chellow/e/hh_parser_simple_csv.py,sha256=RN4QOLvTQeoPrpvXvQ9hkOBZnR5piybLfjCiSJdjpjs,2112
+chellow/e/hh_parser_df2.py,sha256=tRAoVUUoJDlfPopm6usEBnhJz7dXMc2_KEWbkW9Gyq4,4330
+chellow/e/hh_parser_simple_csv.py,sha256=lJx9tw9BWFSoBmns1Cws_vY-OIn90LPt2yvIN_CFcTE,2177
+chellow/e/hh_parser_vital_xlsx.py,sha256=g9-CElfH1PPfwpuUcVvD6WQpBlNxCo8j9pq_0Yza0ZM,4125
 chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
 chellow/e/lcc.py,sha256=OkpynN8_iAdHRlu-yyU6BhRUqYYOZsUnl0HbHULYo_4,4670
 chellow/e/mdd_importer.py,sha256=NugJr2JhuzkPTsEMl_5UdQuw5K2p8lVJ-hyz4MK6Hfg,35762
@@ -38,7 +39,7 @@ chellow/e/system_price.py,sha256=6w5J7bzwFAZubE2zdOFRiS8IIrVP8hkoIOaG2yCt-Ic,623
 chellow/e/tlms.py,sha256=M33D6YpMixu2KkwSCzDRM3kThLgShg8exp63Obo75l8,8905
 chellow/e/tnuos.py,sha256=XseYztPUsQXNKuBmystO2kzzwAG9ehCZgpGBTdgSk-A,4313
 chellow/e/triad.py,sha256=lIQj7EdUrcFwEqleuHZXYU_bfzIwNOqUVVxB3NPQt4A,13710
-chellow/e/views.py,sha256=aljiDbLvO_rUl00ohDDxs-3-xPmsuguaOyo69lcXcBA,220624
+chellow/e/views.py,sha256=LG9KeYfXDkOvd7GrywuQPG1qDnpFvKe1oa95H0wRlqY,220618
 chellow/e/bill_parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/bill_parsers/activity_mop_stark_xlsx.py,sha256=UgWXDPzQkQghyj_lfgBqoSJpHB-t-qOdSaB8qY6GLog,4071
 chellow/e/bill_parsers/annual_mop_stark_xlsx.py,sha256=-HMoIfa_utXYKA44RuC0Xqv3vd2HLeQU_4P0iBUd3WA,4219
@@ -52,7 +53,7 @@ chellow/e/bill_parsers/gdf_csv.py,sha256=ZfK3Oc6oP28p_P9DIevLNB_zW2WLcEJ3Lvb1gL3
 chellow/e/bill_parsers/haven_csv.py,sha256=0uENq8IgVNqdxfBQMBxLTSZWCOuDHXZC0xzk52SbfyE,13652
 chellow/e/bill_parsers/haven_edi.py,sha256=YGPHRxPOhje9s32jqPHHELni2tooOYj3cMC_qaZVPq4,16107
 chellow/e/bill_parsers/haven_edi_tprs.py,sha256=ZVX9CCqUybsot_Z0BEOJPvl9x5kSr7fEWyuJXvZDcz4,11841
-chellow/e/bill_parsers/mm.py,sha256=W_n7xlXrSH6QewkCnNQpMAxWUocXctIHo7XR-t95Aas,9543
+chellow/e/bill_parsers/mm.py,sha256=mXghmH_gLx82O20CUMBnUysAJqh-OMkR_-CGEcKrzhs,9544
 chellow/e/bill_parsers/nonsettlement_dc_stark_xlsx.py,sha256=yogXTuQHGRL7IiqvRWr2C9V24ez1j9Yx0128UygPE_k,4723
 chellow/e/bill_parsers/settlement_dc_stark_xlsx.py,sha256=PlEqCZuJ9DfQXeeYQ64jtf3ML7sUt_tt61QOOTnkE5c,6380
 chellow/e/bill_parsers/sse_edi.py,sha256=L85DOfNkqexeEIEr8pCBn_2sHJI-zEaw6cogpE3YyYM,15204
@@ -60,7 +61,7 @@ chellow/e/bill_parsers/sww_xls.py,sha256=QEjiuvwvr5FuWCfqqVw8LaA_vZyAKsvRAS5fw3x
 chellow/gas/bill_import.py,sha256=w0lPgK_Drzh8rtnEBQe3qFuxrgzZ6qQSgpaGrrGznMU,6549
 chellow/gas/bill_parser_csv.py,sha256=Ecdy-apFT-mWAxddAsM4k1s-9-FpIaOfjP0oFc0rdQg,5557
 chellow/gas/bill_parser_engie_edi.py,sha256=Ko0vZP-QdVQ1uuhS_5cdrii60_cM4b_LFJMoY0pZqnk,8950
-chellow/gas/bill_parser_total_edi.py,sha256=hbjvahC8mQ0RuyIEQVJu18XNqq7T_htnV-OYg6NSuiE,7379
+chellow/gas/bill_parser_total_edi.py,sha256=HdX2ctKvqIA5saf8VkpWmYWfqaZsdB-sAEcMSZjTgE4,7425
 chellow/gas/ccl.py,sha256=DMlcPUELZi00CaDekVJINYk3GgH5apFrImVdmkbyba0,2913
 chellow/gas/cv.py,sha256=4cdYYQ8Qak6NeYdBCB4YaQ0jX8-UkaydIIdibCQuXxM,7344
 chellow/gas/dn_rate_parser.py,sha256=Mq8rAcUEUxIQOks59bsCKl8GrefvoHbrTCHqon9N0z0,11340
@@ -69,7 +70,7 @@ chellow/gas/transportation.py,sha256=Bkg8TWOs-v0ES-4qqwbleiOhqbE_t2KauUx9JYMZELM
 chellow/gas/views.py,sha256=zOlJnL6eVq7Tag-0NMabzCPQk5AoTrclEz-923pbwAE,57851
 chellow/reports/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/reports/report_109.py,sha256=FmRWG8wQC97cj0nqQH7mrfbYesRIKWXQLh6ERHvtHrU,11212
-chellow/reports/report_111.py,sha256=AlYjvCDpGNe1qBuzW52EONWDsFaIVElug0Vpe9qm1PY,28169
+chellow/reports/report_111.py,sha256=rgwoOPfvbfwIC_IjBqs2e8umF_YTtpyejLrX_uCgyHE,28145
 chellow/reports/report_169.py,sha256=tnTiHmhigcgZ7E8rUSfgzsVYUCW947xBZZq04-KqfK8,9287
 chellow/reports/report_181.py,sha256=mCL6vXfrj8r0kL-LilDS0SnSu5tJOzCq7eMJVx9zpEw,4999
 chellow/reports/report_183.py,sha256=DZX-hHJPl_Tbgkt31C_cuLZg_5L2b6iCPQ5foOZizR0,8817
@@ -187,7 +188,7 @@ chellow/templates/e/dc_bill_import.html,sha256=NHjMSoFizvFQIaPWuVE3nTCtMDTzJB0Xm
 chellow/templates/e/dc_bill_imports.html,sha256=lCaUR47r9KPr0VrQhEvVEaKexM5R_nmkxtzgpWZ0e9s,2135
 chellow/templates/e/dc_contract.html,sha256=b6DLDrGdzN9Rri56pFvfpE7QZKYXGHib2veYAztuHlg,2352
 chellow/templates/e/dc_contract_edit.html,sha256=IObmbHQmeZ_LSpnYgabmhoSNUR3aPmm-Jk5nJLM_u74,1706
-chellow/templates/e/dc_contract_hh_import.html,sha256=UODiBFNohb60MjH1w-9JW1JE0O9GR2uKPGw-lD7Da5g,848
+chellow/templates/e/dc_contract_hh_import.html,sha256=JncR3L6cOK4jghsGyr-itEqlIemXBXt3kL09wpqnQSE,856
 chellow/templates/e/dc_contract_hh_imports.html,sha256=eXFDGyzSgag4JRism81_p5yTzQOjCIXaVkQ8tl3dDcM,8172
 chellow/templates/e/dc_contract_properties.html,sha256=2EmA91gYWZcTOHQ3PxXQrpTOb8dm0OGjwRWsW6qNI9s,455
 chellow/templates/e/dc_contract_properties_edit.html,sha256=TGYcHTlWbk5WmjZPkdbHhRVZcOpEfhowCp2penZcGiE,1704
@@ -365,6 +366,6 @@ chellow/templates/g/supply_note_edit.html,sha256=b8mB6_ucBwoljp03iy6AgVaZUhGw3-1
 chellow/templates/g/supply_notes.html,sha256=6epNmZ3NKdXZz27fvmRUGeffg_oc1kmwuBeyRzQe3Rg,854
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1726510224.0.0.dist-info/METADATA,sha256=5yHyg_hOusHvyLO5578hobk7B9yosB2-EFVHM94oE7U,12204
-chellow-1726510224.0.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-chellow-1726510224.0.0.dist-info/RECORD,,
+chellow-1726664509.0.0.dist-info/METADATA,sha256=PhQBcuYRWYyJltXJBei7Jl22-HKANq4qi5Nae4hRNhs,12204
+chellow-1726664509.0.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+chellow-1726664509.0.0.dist-info/RECORD,,