chellow 1707388338.0.0__py3-none-any.whl → 1707995667.0.0__py3-none-any.whl

chellow/dloads.py

@@ -64,6 +64,10 @@ class DloadFile:
         self._check_exists()
         return self.f.truncate(*args, **kwargs)
 
+    def writestr(self, *args, **kwargs):
+        self._check_exists()
+        return self.f.writestr(*args, **kwargs)
+
     def close(self):
         self.f.close()
         self._check_exists()

chellow/e/hh_importer.py

@@ -48,6 +48,7 @@ class HhDataImportProcess(threading.Thread):
         super().__init__(name=f"HH Manual Import: contract {dc_contract_id}")
 
         self.messages = []
+        self.error = None
         self.istream = istream
         self.dc_contract_id = dc_contract_id
         self.id = process_id
@@ -74,14 +75,16 @@ class HhDataImportProcess(threading.Thread):
                 parser_module = importlib.import_module(
                     f"chellow.e.hh_parser_{mod_name}"
                 )
-                self.converter = parser_module.create_parser(self.istream, mpan_map)
+                self.converter = parser_module.create_parser(
+                    self.istream, mpan_map, self.messages
+                )
                 sess.rollback()
                 HhDatum.insert(sess, self.converter, contract)
                 sess.commit()
             except BadRequest as e:
-                self.messages.append(e.description)
+                self.error = e.description
             except BaseException:
-                self.messages.append(f"Outer problem {traceback.format_exc()}")
+                self.error = f"Outer problem {traceback.format_exc()}"
 
     def get_status(self):
         return (
@@ -281,8 +284,8 @@ class HhImportTask(threading.Thread):
             for message in self.importer.messages:
                 self.log(message)
 
-            if len(self.importer.messages) > 0:
-                raise BadRequest("Problem loading file.")
+            if self.importer.error is not None:
+                raise BadRequest(f"Problem loading file. {self.importer.error}")
 
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)
@@ -384,12 +387,12 @@ class HhImportTask(threading.Thread):
 
             self.importer.run()
             messages = self.importer.messages
-            self.importer = None
             for message in messages:
                 self.log(message)
 
-            if len(messages) > 0:
-                raise BadRequest("Problem loading file.")
+            if self.importer.error is not None:
+                raise BadRequest(f"Problem loading file. {self.importer.error}")
+            self.importer = None
 
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)

chellow/e/hh_parser_bg_csv.py

@@ -8,12 +8,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import parse_mpan_core, utc_datetime
 
 
-def create_parser(reader, mpan_map):
-    return HhParserBglobal(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return HhParserBglobal(reader, mpan_map, messages)
 
 
 class HhParserBglobal:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.shredder = zip(itertools.count(1), csv.reader(reader))
         self.line_number, self.values = next(self.shredder)
         self.mpan_map = mpan_map

chellow/e/hh_parser_df2.py

@@ -6,12 +6,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import HH, parse_mpan_core, utc_datetime
 
 
-def create_parser(reader, mpan_map):
-    return StarkDf2HhParser(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return StarkDf2HhParser(reader, mpan_map, messages)
 
 
 class StarkDf2HhParser:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.line = self.line_number = None
         self.reader = zip(itertools.count(1), reader)
 
@@ -25,6 +25,9 @@ class StarkDf2HhParser:
             4: "REACTIVE_EXP",
         }
 
+        self.mpan_map = mpan_map
+        self.messages = messages
+
     def __iter__(self):
         return self
 
@@ -36,6 +39,15 @@ class StarkDf2HhParser:
                 lline = self.line.strip().upper()
                 if lline.startswith("#O"):
                     self.core = parse_mpan_core(lline[2:])
+                    if self.core in self.mpan_map and self.mpan_map[self.core] is None:
+                        msg = f"The MPAN core {self.core} has been ignored"
+                        if len(self.messages) == 0 or self.messages[-1] != msg:
+                            self.messages.append(
+                                f"The MPAN core {self.core} has been ignored"
+                            )
+                        self.core = None
+                        continue
+
                 elif lline.startswith("#S"):
                     sensor = int(lline[2:].strip())
                     try:
@@ -72,6 +84,10 @@ class StarkDf2HhParser:
                     except ValueError:
                         raise BadRequest("Problem parsing the value: " + fields[2])
                     status = fields[3][-1]
+
+                    if self.core is None:
+                        continue
+
                     local_datum = {
                         "mpan_core": self.core,
                         "channel_type": self.channel_type,

chellow/e/hh_parser_simple_csv.py

@@ -8,12 +8,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import parse_channel_type, parse_mpan_core, to_utc, validate_hh_start
 
 
-def create_parser(reader, mpan_map):
-    return HhParserCsvSimple(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return HhParserCsvSimple(reader, mpan_map, messages)
 
 
 class HhParserCsvSimple:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.shredder = zip(itertools.count(1), csv.reader(reader))
         next(self.shredder)  # skip the title line
         self.values = None

chellow/reports/report_183.py

@@ -1,10 +1,8 @@
 import csv
-import os
 import sys
 import threading
 import traceback
 from io import StringIO
-from zipfile import ZipFile
 
 from flask import g, request
 
@@ -12,7 +10,7 @@ from sqlalchemy import or_, select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import null, true
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import Era, RSession, Site, SiteEra, Supply, User
 from chellow.utils import (
     csv_make_val,
@@ -112,7 +110,6 @@ def none_content(site_codes, typ, start_date, finish_date, user_id, file_name):
     try:
         with RSession() as rsess:
             user = User.get_by_id(rsess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(file_name, user)
             sites_q = (
                 select(Site)
                 .join(SiteEra)
@@ -127,7 +124,7 @@ def none_content(site_codes, typ, start_date, finish_date, user_id, file_name):
             if site_codes is not None:
                 sites_q = sites_q.where(Site.code.in_(site_codes))
 
-            zf = ZipFile(running_name, "w")
+            zf = open_file(file_name, user, mode="w", is_zip=True)
 
             for site in rsess.execute(sites_q).scalars():
                 _process_site(rsess, zf, site, start_date, finish_date, typ)
@@ -141,7 +138,6 @@ def none_content(site_codes, typ, start_date, finish_date, user_id, file_name):
     finally:
         if zf is not None:
             zf.close()
-            os.rename(running_name, finished_name)
 
 
 def site_content(site_id, start_date, finish_date, user_id, file_name):
@@ -149,8 +145,7 @@ def site_content(site_id, start_date, finish_date, user_id, file_name):
     try:
         with RSession() as rsess:
             user = User.get_by_id(rsess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(file_name, user)
-            f = open(running_name, mode="w", newline="")
+            f = open_file(file_name, user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             site = Site.get_by_id(rsess, site_id)
             sites = rsess.query(Site).filter(Site.id == site_id)
@@ -229,7 +224,6 @@ def site_content(site_id, start_date, finish_date, user_id, file_name):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_post(sess):

chellow/templates/e/dc_contract_edit.html

@@ -62,7 +62,11 @@
 	"hostname": "example.com",
 	"username": "username",
 	"password": "password",
-	"directories": ["downloads1", "downloads2"]}
+	"directories": ["downloads1", "downloads2"]
+	"mpan_map": {  \\ Optional
+	  "99 0993 2821 985": null,  \\ Ignore MPAN
+	},
+}
 			</pre>
 		</code>
 		<p>For the HTTPS protocol:</p>
@@ -81,6 +85,9 @@
       "name2": val2,
     }
   }
+	"mpan_map": {  \\ Optional
+	  "99 0993 2821 985": null,  \\ Ignore MPAN
+	},
 }
 
 				{% endraw %}

{chellow-1707388338.0.0.dist-info → chellow-1707995667.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1707388338.0.0
+Version: 1707995667.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1707388338.0.0.dist-info → chellow-1707995667.0.0.dist-info}/RECORD

@@ -2,7 +2,7 @@ chellow/__init__.py,sha256=_9hq4MlC2YB7kUh4iH9hwmx7BqMV6JeyvGVB8uWk8Cs,10127
 chellow/api.py,sha256=mk17TfweR76DPFC8lX2SArTjai6y6YshASxqO1w-_-s,11036
 chellow/bank_holidays.py,sha256=22EdNPvUQG_hFt82OKkeKzRFht78Urhu0Sc6Kpj-1wo,4488
 chellow/commands.py,sha256=ESBe9ZWj1c3vdZgqMZ9gFvYAB3hRag2R1PzOwuw9yFo,1302
-chellow/dloads.py,sha256=ce8EtOt5GeMczc0RWO8YaUv6cLt9MzIwXaeCsf_R7lw,5090
+chellow/dloads.py,sha256=vtebrKchBt_oANDO_c-aL2jO-w1KKwBcYaG_VgzUlJI,5209
 chellow/edi_lib.py,sha256=het3R0DBGtXEo-Sy1zvXQuX9EWQ1X6Y33G4l1hYYuds,51859
 chellow/general_import.py,sha256=4LduwzNipxOiHYHQLb0x96_m3RPuDnQMbJzOFDbOqTg,65039
 chellow/models.py,sha256=7EX1oRpIcVRsjpzIC4ROOr-zp1-C7kdoj3WSNp2AKr0,236400
@@ -24,10 +24,10 @@ chellow/e/dno_rate_parser.py,sha256=5MYEbcYngh7n6Szn2h9zLTwQKXH-WHKy6c80PYwNLLQ,
 chellow/e/duos.py,sha256=ISTcNqe9KNjVNM2Qs8IBoQxnmSXOt5W_G7tZxp4T78M,28870
 chellow/e/elexon.py,sha256=s8QdLUQMeylLhghxvvhLyOL3-0ifUnMfmo3vcbot_DY,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
-chellow/e/hh_importer.py,sha256=G1CK3AQQuJaNXCxGEY3UU_CjqOzfVZT-PDR67b3zpA0,20362
-chellow/e/hh_parser_bg_csv.py,sha256=IOjM-QliKzvSXyhslmxz504XWnsGkVKEaCQ48SNeNeQ,2423
-chellow/e/hh_parser_df2.py,sha256=G4kt4N4mNjUa1Z7B1Ov6rAyZmWDyGEVFYscdghToMRo,3558
-chellow/e/hh_parser_simple_csv.py,sha256=lVRprIEjDpTQdeXm2xcLmRAyre2LWec8ueVwcCISZPo,2082
+chellow/e/hh_importer.py,sha256=nhmlWDNBm8U4salBud8ivdaDGF1lkY82NqQCrGuPYRI,20483
+chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
+chellow/e/hh_parser_df2.py,sha256=ynNNRMpLm9qs23tJjfm0vGSaAduXfPMw_7-WfQHXliQ,4209
+chellow/e/hh_parser_simple_csv.py,sha256=RN4QOLvTQeoPrpvXvQ9hkOBZnR5piybLfjCiSJdjpjs,2112
 chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
 chellow/e/lcc.py,sha256=q5iGbeYZdnozkIdbGuxXoJO-1KDO7AHqYOXEGBIaNHI,4963
 chellow/e/mdd_importer.py,sha256=9GN-at0DC3vOjd-0N2U856O3hInnjkMWWcRfxzQ5DjA,32042
@@ -71,7 +71,7 @@ chellow/reports/report_111.py,sha256=qU5MYPQzZ_cEvuERE8bxq93D-dN1yd4oML6FIqd0Jdk
 chellow/reports/report_169.py,sha256=tnTiHmhigcgZ7E8rUSfgzsVYUCW947xBZZq04-KqfK8,9287
 chellow/reports/report_177.py,sha256=JPfmdJMxRqzV_jpXjH-ziuRMwdK5UIjBUhRETJ2PnvA,10505
 chellow/reports/report_181.py,sha256=Fm2kgqjZvTVU31H6KVBVg3bqcGPHomJG82KXa7I7k4E,4989
-chellow/reports/report_183.py,sha256=ZEX1TuL9Pho7-HAJ1BBNMsR9w3Y7ryj78Kyrc_Bw-zI,9081
+chellow/reports/report_183.py,sha256=DZX-hHJPl_Tbgkt31C_cuLZg_5L2b6iCPQ5foOZizR0,8817
 chellow/reports/report_187.py,sha256=UvpaYHjyJFNV5puYq8_KxfzoBtVrwFgIGUOmC5oGA9A,9956
 chellow/reports/report_215.py,sha256=_HZO_MhL51ttzmG6IrT8YVG6WmJMkRNWsXOW7AL7GWs,6766
 chellow/reports/report_219.py,sha256=o2eEg3mX9_raP2b4gjc2Gu4vqnMqcvvJBYQ1oQjxvpE,6637
@@ -186,7 +186,7 @@ chellow/templates/e/dc_bill_edit.html,sha256=0GsN-ZIc4q-z_xs8igC2ZS6t4soo2SvB3qR
 chellow/templates/e/dc_bill_import.html,sha256=NHjMSoFizvFQIaPWuVE3nTCtMDTzJB0XmH8jXfV1tiA,2188
 chellow/templates/e/dc_bill_imports.html,sha256=lCaUR47r9KPr0VrQhEvVEaKexM5R_nmkxtzgpWZ0e9s,2135
 chellow/templates/e/dc_contract.html,sha256=_fq_s69kmbrGlScYngRYm4cPdRgh2roMvSfvG0J4xzA,2326
-chellow/templates/e/dc_contract_edit.html,sha256=BQPhcFqlBGaxBqinqxoeGZQPA6nmgfNAo9EcaOtycjM,2960
+chellow/templates/e/dc_contract_edit.html,sha256=wKK2pXN78VfSJ6gjfNRkikRFkYTiKtGL3UeVFIYSRwc,3115
 chellow/templates/e/dc_contract_hh_import.html,sha256=UODiBFNohb60MjH1w-9JW1JE0O9GR2uKPGw-lD7Da5g,848
 chellow/templates/e/dc_contract_hh_imports.html,sha256=eXFDGyzSgag4JRism81_p5yTzQOjCIXaVkQ8tl3dDcM,8172
 chellow/templates/e/dc_contracts.html,sha256=v8czpEcMzndngfWNXIKXib6Xrz9OwJjQAmNQNrQ6Y08,1705
@@ -363,6 +363,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1707388338.0.0.dist-info/METADATA,sha256=AhyvitvJT46wtS3-60PA83X_l7kV0qbGN-jbpIcDBSk,12203
-chellow-1707388338.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-chellow-1707388338.0.0.dist-info/RECORD,,
+chellow-1707995667.0.0.dist-info/METADATA,sha256=qAtfNaDLJjw2jkahNM2b5loL8f-MkeEf2NXOo2tun_E,12203
+chellow-1707995667.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+chellow-1707995667.0.0.dist-info/RECORD,,