chellow 1726510224.0.0__tar.gz → 1726664509.0.0__tar.gz

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1726510224.0.0
+Version: 1726664509.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1726664509.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1726664509.0.0"

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/e/bill_parsers/mm.py

@@ -217,7 +217,7 @@ def _handle_0860(headers, pre_record, record):
 
 def _handle_1455(headers, pre_record, record):
     parts = _chop_record(
-        record, ccl_kwh=13, unknown_1=8, ccl_rate=15, ccl_gbp=12, unkown_2=8
+        record, ccl_kwh=13, unknown_1=10, ccl_rate=13, ccl_gbp=12, unkown_2=8
     )
     bd = headers["breakdown"]
     bd["ccl-kwh"] += Decimal(parts["ccl_kwh"])

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/e/hh_importer.py

@@ -8,7 +8,7 @@ import traceback
 from collections import defaultdict, deque
 from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal
-from io import StringIO, TextIOWrapper
+from io import StringIO
 from pathlib import Path
 
 import jinja2
@@ -40,7 +40,7 @@ from chellow.utils import (
 processes = defaultdict(list)
 tasks = {}
 
-extensions = [".df2", ".simple.csv", ".bg.csv"]
+extensions = [".df2", ".simple.csv", ".bg.csv", ".vital.xlsx"]
 
 
 class HhDataImportProcess(threading.Thread):
@@ -277,7 +277,7 @@ class HhImportTask(threading.Thread):
             self.log(f"File size is {fsize} bytes.")
             self.log(f"Treating files as type {file_type}")
             self.importer = HhDataImportProcess(
-                self.contract_id, 0, TextIOWrapper(f, "utf8"), fpath + file_type, fsize
+                self.contract_id, 0, f, fpath + file_type, fsize
             )
 
             self.importer.run()
@@ -412,7 +412,7 @@ class HhImportTask(threading.Thread):
             self.log(f"File size is {fsize} bytes.")
             self.log(f"Treating files as type {file_type}")
             self.importer = HhDataImportProcess(
-                self.contract_id, 0, TextIOWrapper(f, "utf8"), fpath + file_type, fsize
+                self.contract_id, 0, f, fpath + file_type, fsize
             )
 
             self.importer.run()

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/e/hh_parser_df2.py

@@ -1,5 +1,6 @@
 import decimal
 import itertools
+from codecs import iterdecode
 
 from werkzeug.exceptions import BadRequest
 
@@ -13,7 +14,7 @@ def create_parser(reader, mpan_map, messages):
 class StarkDf2HhParser:
     def __init__(self, reader, mpan_map, messages):
         self.line = self.line_number = None
-        self.reader = zip(itertools.count(1), reader)
+        self.reader = zip(itertools.count(1), iterdecode(reader, "utf-8"))
 
         self.line_number, self.line = next(self.reader)
         if self.line.strip().upper() != "#F2":

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/e/hh_parser_simple_csv.py

@@ -1,5 +1,6 @@
 import csv
 import itertools
+from codecs import iterdecode
 from datetime import datetime as Datetime
 from decimal import Decimal
 
@@ -14,7 +15,8 @@ def create_parser(reader, mpan_map, messages):
 
 class HhParserCsvSimple:
     def __init__(self, reader, mpan_map, messages):
-        self.shredder = zip(itertools.count(1), csv.reader(reader))
+        s = iterdecode(reader, "utf-8")
+        self.shredder = zip(itertools.count(1), csv.reader(s))
         next(self.shredder)  # skip the title line
         self.values = None
 

chellow-1726664509.0.0/chellow/e/hh_parser_vital_xlsx.py

@@ -0,0 +1,132 @@
+from datetime import datetime as Datetime
+
+from decimal import Decimal, InvalidOperation
+from openpyxl import load_workbook
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import HH, ct_datetime, hh_before, hh_range, to_utc
+
+
+def create_parser(input_stream, mpan_map, messages):
+    return HhParserVital(input_stream, mpan_map, messages)
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_date(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    val = cell.value
+    if not isinstance(val, Datetime):
+        raise BadRequest(
+            f"Problem reading {val} (of type {type(val)}) as a timestamp at "
+            f"{cell.coordinate}."
+        )
+    return val
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_dec(sheet, col, row):
+    return get_dec_from_cell(get_cell(sheet, col, row))
+
+
+def get_dec_from_cell(cell):
+    try:
+        return Decimal(str(cell.value))
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+def find_hhs(sheet, imp_mpan, exp_mpan):
+    row = pres_readings = pres_ts = None
+    caches = {}
+    try:
+        for row in range(2, len(sheet["A"]) + 1):
+            imp_read_cell = get_cell(sheet, "H", row)
+            imp_read_val = imp_read_cell.value
+            if imp_read_val is None or imp_read_val == "":
+                break
+
+            exp_read_cell = get_cell(sheet, "I", row)
+            exp_read_val = exp_read_cell.value
+            if exp_read_val is None or exp_read_val == "":
+                break
+
+            # '16/09/2024 14:45:00
+            ts_str = get_str(sheet, "A", row)
+            ts_naive = Datetime.strptime(ts_str, "%d/%m/%Y %H:%M:%S")
+            ts_ct = ct_datetime(
+                ts_naive.year,
+                ts_naive.month,
+                ts_naive.day,
+                ts_naive.hour,
+                ts_naive.minute,
+            )
+
+            imp_read = get_dec_from_cell(imp_read_cell)
+            exp_read = get_dec_from_cell(exp_read_cell)
+            ts = to_utc(ts_ct)
+
+            if ts_naive.minute in (0, 30) and hh_before(ts, pres_ts):
+                if pres_readings is not None:
+                    pres_imp_read, pres_exp_read = pres_readings
+                    hhs = hh_range(caches, ts, pres_ts - HH)
+                    num_hhs = len(hhs)
+                    val_imp = (pres_imp_read - imp_read) / num_hhs
+                    val_exp = (pres_exp_read - exp_read) / num_hhs
+                    for hh in hhs:
+
+                        yield {
+                            "mpan_core": imp_mpan,
+                            "channel_type": "ACTIVE",
+                            "start_date": hh,
+                            "value": val_imp,
+                            "status": "A",
+                        }
+                        yield {
+                            "mpan_core": exp_mpan,
+                            "channel_type": "ACTIVE",
+                            "start_date": hh,
+                            "value": val_exp,
+                            "status": "A",
+                        }
+
+                pres_readings = imp_read, exp_read
+                pres_ts = ts
+
+    except BadRequest as e:
+        e.description = (f"Problem at row: {row}: {e.description}",)
+        raise e
+
+
+class HhParserVital:
+    def __init__(self, input_stream, mpan_map, messages):
+        book = load_workbook(input_stream, data_only=True)
+        sheet = book.worksheets[0]
+        imp_name = get_str(sheet, "H", 1).strip()
+        imp_mpan = mpan_map[imp_name]
+        exp_name = get_str(sheet, "I", 1).strip()
+        exp_mpan = mpan_map[exp_name]
+        self.data = iter(find_hhs(sheet, imp_mpan, exp_mpan))
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return next(self.data)
+
+    def close(self):
+        pass

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/e/views.py

@@ -3,7 +3,7 @@ import os
 import threading
 from collections import defaultdict
 from datetime import datetime as Datetime
-from io import StringIO
+from io import BytesIO, StringIO
 from itertools import chain, islice
 from random import random
 
@@ -1233,7 +1233,7 @@ def dc_contracts_hh_imports_post(contract_id):
         contract = Contract.get_dc_by_id(g.sess, contract_id)
 
         file_item = req_file("import_file")
-        f = StringIO(str(file_item.stream.read(), "utf-8"))
+        f = BytesIO(file_item.stream.read())
         f.seek(0, os.SEEK_END)
         file_size = f.tell()
         f.seek(0)

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/gas/bill_parser_total_edi.py

@@ -110,10 +110,10 @@ def _process_CCD2(elements, headers):
     units_billed = to_decimal(nuct) / units_divisor
     breakdown[key] += units_billed
 
-    if "start_date" not in headers:
+    if tpref == "standing" or "start_date" not in headers:
         headers["start_date"] = to_date(csdt[0])
 
-    if "finish_date" not in headers:
+    if tpref == "standing" or "finish_date" not in headers:
         headers["finish_date"] = _to_finish_date(cedt[0])
 
     if "mprn" not in headers:

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/reports/report_111.py

@@ -2,7 +2,7 @@ import csv
 import sys
 import threading
 import traceback
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
 from datetime import datetime as Datetime
 from decimal import Decimal
 from itertools import combinations
@@ -11,7 +11,7 @@ from dateutil.relativedelta import relativedelta
 
 from flask import g, request
 
-from sqlalchemy import or_
+from sqlalchemy import or_, select
 from sqlalchemy.orm import joinedload, subqueryload
 from sqlalchemy.sql.expression import null, true
 
@@ -27,7 +27,6 @@ from chellow.models import (
     Contract,
     Era,
     Llfc,
-    MarketRole,
     MtcParticipant,
     RSession,
     RegisterRead,
@@ -380,19 +379,19 @@ def _process_supply(
         while enlarged:
             enlarged = False
             covered_elems = find_elements(bill)
-            covered_bills = OrderedDict(
+            covered_bills = dict(
                 (b.id, b)
-                for b in sess.query(Bill)
-                .join(Batch)
-                .join(Contract)
-                .join(MarketRole)
-                .filter(
-                    Bill.supply == supply,
-                    Bill.start_date <= covered_finish,
-                    Bill.finish_date >= covered_start,
-                    MarketRole.code == market_role_code,
+                for b in sess.scalars(
+                    select(Bill)
+                    .join(Batch)
+                    .where(
+                        Bill.supply == supply,
+                        Bill.start_date <= covered_finish,
+                        Bill.finish_date >= covered_start,
+                        Batch.contract == contract,
+                    )
+                    .order_by(Bill.start_date, Bill.issue_date)
                 )
-                .order_by(Bill.start_date, Bill.issue_date)
             )
             while True:
                 to_del = None

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/chellow/templates/e/dc_contract_hh_import.html

@@ -21,15 +21,16 @@
 				Reached line number {{process.converter.line_number}}.
 			{% endif %}
 		</p>
-	{% elif process.messages|length == 0 %}
+	{% elif process.error is none %}
 		<p>The import has completed successfully.</p>
 	{% else %}
-		<p>The import failed:</p>
-		<ul>
-			{% for message in process.messages %}
-				<li>{{message}}</li>
-			{% endfor %}
-		</ul>
+		<p>The import failed. {{process.error}}</p>
 	{% endif %}
+	
+	<ul>
+		{% for message in process.messages %}
+			<li>{{message}}</li>
+		{% endfor %}
+	</ul>
 
 {% endblock %}

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/test/e/bill_parsers/test_bill_parser_mm.py

@@ -96,8 +96,8 @@ def test_handle_1455():
     record = "".join(
         (
             "0000000000034",  # ccl_kwh=13
-            "xxxxxxxx",  # unknown_1=8
-            "000000000003.00",  # ccl_rate=15
+            "xxxxxxxxxx",  # unknown_1=8
+            "0000000003.00",  # ccl_rate=15
             "000000000876",  # ccl_gbp=12
             "0       ",  # unknown_2=8
         )

{chellow-1726510224.0.0 → chellow-1726664509.0.0}/test/e/test_hh_parser_df2.py

@@ -1,11 +1,12 @@
 from decimal import Decimal
+from io import BytesIO
 
 from chellow.e.hh_parser_df2 import create_parser
 from chellow.utils import utc_datetime
 
 
 def test_parser():
-    reader = [
+    reader_lines = [
         "#F2",
         "#O 2248274887874-jlkkk38",
         "#S 1",
@@ -13,6 +14,7 @@ def test_parser():
         "16/04/2024,01:00,0.064,A",
         "16/04/2024,01:30,0,A",
     ]
+    reader = BytesIO("\n".join(reader_lines).encode("utf8"))
     mpan_map = {}
     messages = []
 

chellow-1726664509.0.0/test/e/test_hh_parser_vital_xlsx.py

@@ -0,0 +1,61 @@
+from decimal import Decimal
+
+from openpyxl import Workbook
+
+from chellow.e.hh_parser_vital_xlsx import find_hhs
+
+from chellow.utils import utc_datetime
+
+
+def test_find_hhs():
+    wb = Workbook()
+    sheet = wb.worksheets[0]
+
+    for row, (ts, imp, exp) in enumerate(
+        (
+            ("15/09/2024 21:30:00", 508, 22),
+            ("15/09/2024 21:15:00", 500, 20),
+            ("15/09/2024 21:00:00", 497, 20),
+            ("15/09/2024 20:45:00", 490, 17),
+            ("15/09/2024 20:30:01", 481, 15),
+            ("15/09/2024 20:30:00", 475, 10),
+        ),
+        start=2,
+    ):
+        sheet[f"A{row}"] = ts
+        sheet[f"H{row}"] = Decimal(imp)
+        sheet[f"I{row}"] = Decimal(exp)
+
+    hhs = list(find_hhs(sheet, "99 0000 0000 000", "99 0000 0000 010"))
+    expected_hhs = [
+        {
+            "channel_type": "ACTIVE",
+            "mpan_core": "99 0000 0000 000",
+            "start_date": utc_datetime(2024, 9, 15, 20, 0),
+            "status": "A",
+            "value": Decimal("11"),
+        },
+        {
+            "channel_type": "ACTIVE",
+            "mpan_core": "99 0000 0000 010",
+            "start_date": utc_datetime(2024, 9, 15, 20, 0),
+            "status": "A",
+            "value": Decimal("2"),
+        },
+        {
+            "channel_type": "ACTIVE",
+            "mpan_core": "99 0000 0000 000",
+            "start_date": utc_datetime(2024, 9, 15, 19, 30),
+            "status": "A",
+            "value": Decimal("16"),
+        },
+        {
+            "channel_type": "ACTIVE",
+            "mpan_core": "99 0000 0000 010",
+            "start_date": utc_datetime(2024, 9, 15, 19, 30),
+            "status": "A",
+            "value": Decimal("5"),
+        },
+    ]
+
+    assert hhs == expected_hhs

chellow-1726510224.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1726510224.0.0"