chellow 1731581146.0.0__py3-none-any.whl → 1733390302.0.0__py3-none-any.whl

chellow/e/bill_parsers/mm.py

@@ -147,6 +147,7 @@ READ_TYPE_LOOKUP = {
     " ": "E",
     "E": "E",
     "N": "N",
+    "S": "N",
 }
 
 

chellow/e/hh_parser_df2.py

@@ -4,7 +4,12 @@ from codecs import iterdecode
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import HH, parse_mpan_core, utc_datetime
+from chellow.utils import HH, hh_format, parse_mpan_core, utc_datetime
+
+
+def _append_new_message(messages, message):
+    if len(messages) == 0 or messages[-1] != message:
+        messages.append(message)
 
 
 def create_parser(reader, mpan_map, messages):
@@ -41,14 +46,6 @@ class StarkDf2HhParser:
                 if lline.startswith("#O"):
                     outstation_parts = lline[2:].split("-")
                     self.core = parse_mpan_core(outstation_parts[0])
-                    if self.core in self.mpan_map and self.mpan_map[self.core] is None:
-                        msg = f"The MPAN core {self.core} has been ignored"
-                        if len(self.messages) == 0 or self.messages[-1] != msg:
-                            self.messages.append(
-                                f"The MPAN core {self.core} has been ignored"
-                            )
-                        self.core = None
-                        continue
 
                 elif lline.startswith("#S"):
                     sensor = int(lline[2:].strip())
@@ -87,11 +84,31 @@ class StarkDf2HhParser:
                         raise BadRequest("Problem parsing the value: " + fields[2])
                     status = fields[3][-1]
 
-                    if self.core is None:
-                        continue
+                    mpan_core = self.core
+                    if self.core in self.mpan_map:
+                        mapping = self.mpan_map[self.core]
+                        from_date = mapping["from"]
+                        to_date = mapping["to"]
+                        if from_date <= start_date <= to_date:
+                            mpan_core = mapping["mpan"]
+
+                            if mpan_core is None:
+                                _append_new_message(
+                                    self.messages,
+                                    f"The MPAN core {self.core} is ignored from "
+                                    f"{hh_format(from_date)} to {hh_format(to_date)}",
+                                )
+                                continue
+                            else:
+                                _append_new_message(
+                                    self.messages,
+                                    f"The MPAN core {self.core} is changed to "
+                                    f"{mpan_core} from {hh_format(from_date)} to "
+                                    f"{hh_format(to_date)}",
+                                )
 
                     local_datum = {
-                        "mpan_core": self.core,
+                        "mpan_core": mpan_core,
                         "channel_type": self.channel_type,
                         "start_date": start_date,
                         "value": value,

chellow/general_import.py

@@ -1704,21 +1704,43 @@ for k in tuple(globals().keys()):
         typ_funcs[k[len(PREFIX) :]] = globals()[k]
 
 
+def _process_line(sess, hh_data, line, args):
+
+    action = add_arg(args, "action", line, 0).lower()
+    if action not in ALLOWED_ACTIONS:
+        raise BadRequest(f"The 'action' field must be one of {ALLOWED_ACTIONS}")
+    typ = add_arg(args, "type", line, 1).lower()
+    vals = line[2:]
+    if typ == "hh_datum":
+        if action == "insert":
+            hh_data.append(
+                {
+                    "mpan_core": parse_mpan_core(add_arg(args, "MPAN Core", vals, 0)),
+                    "start_date": parse_hh_start(add_arg(args, "Start Date", vals, 1)),
+                    "channel_type": parse_channel_type(
+                        add_arg(args, "Channel Type", vals, 2)
+                    ),
+                    "value": Decimal(add_arg(args, "Value", vals, 3)),
+                    "status": add_arg(args, "Status", vals, 4),
+                }
+            )
+    else:
+        try:
+            typ_func = typ_funcs[typ]
+            typ_func(sess, action, vals, args)
+        except KeyError:
+            raise BadRequest(f"The type {typ} is not recognized.")
+
+
 class GeneralImporter(threading.Thread):
     def __init__(self, f):
         threading.Thread.__init__(self)
         self.line_number = None
         self.f = f
-        self.rd_lock = threading.Lock()
         self.error_message = None
         self.args = []
-        self.hh_data = []
-
-    def get_fields(self):
-        fields = {"line_number": self.line_number, "error_message": self.error_message}
-        if self.error_message is not None:
-            fields["csv_line"] = self.args
-        return fields
+        self.failed_args = []
+        self.failed_lines = []
 
     def run(self):
         try:
@@ -1735,63 +1757,32 @@ class GeneralImporter(threading.Thread):
                     if len(line) > 0 and line[0].startswith("#"):
                         continue
 
-                    action = add_arg(self.args, "action", line, 0).lower()
-                    if action not in ALLOWED_ACTIONS:
-                        raise BadRequest(
-                            f"The 'action' field must be one of {ALLOWED_ACTIONS}"
-                        )
-                    typ = add_arg(self.args, "type", line, 1).lower()
-                    vals = line[2:]
-                    if typ == "hh_datum":
-                        if action == "insert":
-                            hh_data.append(
-                                {
-                                    "mpan_core": parse_mpan_core(
-                                        add_arg(self.args, "MPAN Core", vals, 0)
-                                    ),
-                                    "start_date": parse_hh_start(
-                                        add_arg(self.args, "Start Date", vals, 1)
-                                    ),
-                                    "channel_type": parse_channel_type(
-                                        add_arg(self.args, "Channel Type", vals, 2)
-                                    ),
-                                    "value": Decimal(
-                                        add_arg(self.args, "Value", vals, 3)
-                                    ),
-                                    "status": add_arg(self.args, "Status", vals, 4),
-                                }
-                            )
-                    else:
-                        try:
-                            typ_func = typ_funcs[typ]
-                            typ_func(sess, action, vals, self.args)
-                        except KeyError:
-                            raise BadRequest(f"The type {typ} is not recognized.")
+                    try:
+                        with sess.begin_nested():
+                            _process_line(sess, hh_data, line, self.args)
+
+                    except BadRequest as e:
+                        self.args.insert(0, ("Error", e.description))
+                        self.failed_args.append(self.args)
+                        self.failed_lines.append(line)
 
                 HhDatum.insert(sess, hh_data)
-                sess.commit()
+
+                if len(self.failed_lines) == 0:
+                    sess.commit()
+                else:
+                    sess.rollback()
         except BadRequest as e:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = e.description
-            finally:
-                self.rd_lock.release()
+            self.error_message = e.description
         except BaseException:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = traceback.format_exc()
-            finally:
-                self.rd_lock.release()
+            self.error_message = traceback.format_exc()
 
 
 def start_process(f):
-    try:
+    with process_lock:
         global process_id
-        process_lock.acquire()
         proc_id = process_id
         process_id += 1
-    finally:
-        process_lock.release()
 
     process = GeneralImporter(f)
     processes[proc_id] = process
@@ -1800,16 +1791,9 @@ def start_process(f):
 
 
 def get_process_ids():
-    try:
-        process_lock.acquire()
+    with process_lock:
         return processes.keys()
-    finally:
-        process_lock.release()
 
 
 def get_process(id):
-    try:
-        process_lock.acquire()
-        return processes[id]
-    finally:
-        process_lock.release()
+    return processes[id]

chellow/templates/e/dc_contract_properties_edit.html

@@ -39,8 +39,17 @@
   "private_key": "private_key", /* Remove if using password */
   "directories": ["downloads1", "downloads2"],
   "mpan_map": {  /* Optional */
-    "99 0993 2821 985": null,  /* Ignore MPAN */
-  },
+		"99 0993 2821 985": {
+			"mpan": null,  /* Ignore MPAN */
+			"from": 2024-12-04T10:00:00Z,
+			"to": 2025-01-12:00:00Z,
+		},
+		"99 0873 1942 952": {
+			"mpan": "99 7821 8541 389",
+			"from": 2024-11-16T22:00:00Z,
+			"to": 2025-02-18:00:00Z,
+		},
+	},
 }
 		</pre>
 	</code>

chellow/templates/general_import.html

@@ -1,47 +1,64 @@
 {% extends "base.html" %}
 
 {% block title %}
-	» General Imports » {{process_id}}
+	» General Imports » {{import_id}}
 {% endblock %}
 
 {% block nav %}
 	<a href="/general_imports">General Imports</a> &raquo;
-	{{process_id}}
+	{{import_id}}
 {% endblock %}
 
 {% block content %}
-	{% if line_number %}
-		<p>Line number {{line_number}}.</p>
+	{% if importer.line_number %}
+		<p>Line number {{importer.line_number}}.</p>
 	{% endif %}
 
-	{% if is_alive %}
+	{% if importer.is_alive() %}
 			<p>Still running. Refresh the page to see latest progress.</p>
 	{% else %}
-			{% if error_message %}
-					<p>
-						The import has failed and so the whole import has been rolled
-						back. The reason for the failure is:</p>
-					<pre>{{error_message}}</pre>
+		{% if importer.failed_args|length > 0 %}
+				<p>
+					The import has failed and so the whole import has been rolled back.
+				</p>
+
+				<h3>Failed Lines</h3>
+
+				{% for line in importer.failed_args %}
+
+					<table>
+						<tbody>
+							<tr>
+								{% for title, value in line %}
+									<th>{{title}}</th>
+								{% endfor %}
+							</tr>
+							<tr>
+								{% for title, value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						</tbody>
+					</table>
+				{% endfor %}
+
+				<h3>Failed Input</h3>
+
+				<table>
+					<tbody>
+						{% for line in importer.failed_lines %}
+							<tr>
+								{% for value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+			{% elif importer.error_message %}
+				<p>{{importer.error_message}}</p>
 			{% else %}
-					<p>The file has been imported successfully.</p>
+				<p>The file has been imported successfully.</p>
 			{% endif %}
 	{% endif %}
-
-	{% if csv_line %}
-		<table>
-			<caption>Failed Line</caption>
-			<tbody>
-				<tr>
-					{% for title, value in csv_line %}
-						<th>{{title}}</th>
-					{% endfor %}
-				</tr>
-				<tr>
-					{% for title, value in csv_line %}
-						<td>{{value}}</td>
-					{% endfor %}
-				</tr>
-			</tbody>
-		</table>
-	{% endif %}
 {% endblock %}

chellow/views.py

@@ -755,14 +755,13 @@ def general_imports_post():
 @home.route("/general_imports/<int:import_id>")
 def general_import_get(import_id):
     try:
-        proc = chellow.general_import.get_process(import_id)
-        fields = proc.get_fields()
-        fields["is_alive"] = proc.is_alive()
-        fields["process_id"] = import_id
-        return render_template("general_import.html", **fields)
+        importer = chellow.general_import.get_process(import_id)
+        return render_template(
+            "general_import.html", import_id=import_id, importer=importer
+        )
     except BadRequest as e:
         flash(e.description)
-        return render_template("general_import.html", process_id=import_id)
+        return render_template("general_import.html", import_id=import_id)
 
 
 @home.route("/edi_viewer")

{chellow-1731581146.0.0.dist-info → chellow-1733390302.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1731581146.0.0
+Version: 1733390302.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1731581146.0.0.dist-info → chellow-1733390302.0.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ chellow/bank_holidays.py,sha256=T_utYMwe_g1dz5X-aOTdIPryg49SvB7QsWM1yphlqG8,4423
 chellow/commands.py,sha256=ESBe9ZWj1c3vdZgqMZ9gFvYAB3hRag2R1PzOwuw9yFo,1302
 chellow/dloads.py,sha256=dixp-O0MF2_mlwrnKx3D9DH09Qu05BjTo0rZfigTjR4,5534
 chellow/edi_lib.py,sha256=alu20x9ZX06iPfnNI9dEJzuP6RIf4We3Y_M_bl7RrcY,51789
-chellow/general_import.py,sha256=5kJszUnk3CVJclsW-mCjLF0z8lw7CaNNhIzUoaq20kY,65965
+chellow/general_import.py,sha256=tcYZK2N7HOFJSkYyy3MMxVBEvMFjfB9pjF0ksujtkFA,65113
 chellow/models.py,sha256=9LLROzS6gXZGICk0ITXhCyTtKaIY7HW4bMqWg4Mc-xw,244221
 chellow/national_grid.py,sha256=czwIZqzJndSGhEMQ5YzI6hRBhvjkM6VRVYXybf4_KXg,4377
 chellow/proxy.py,sha256=cVXIktPlX3tQ1BYcwxq0nJXKE6r3DtFTtfFHPq55HaM,1351
@@ -12,7 +12,7 @@ chellow/rate_server.py,sha256=fg-Pf_9Hk3bXmC9riPQNGQxBvLvBa_WtNYdwDCjnCSg,5678
 chellow/rrun.py,sha256=1Kt2q_K9UoDG_nsZz-Q6XJiMNKroWqlqFdxn2M6Q8CA,2088
 chellow/testing.py,sha256=Od4HHH6pZrhJ_De118_F55RJEKmAvhUH2S24QE9qFQk,3635
 chellow/utils.py,sha256=32qPWEGzCPKPhSM7ztpzcR6ZG74sVZanScDIdK50Rq4,19308
-chellow/views.py,sha256=eb7WUqqu7LmAX5m_9ICJFVpo0i8WZ9GV89AkqRA3aac,84383
+chellow/views.py,sha256=qJrvhVjb3KL_sHDQl5dLhi0aqUe_IUYhrjiPjdk4lQI,84317
 chellow/e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/aahedc.py,sha256=d2usudp7KYWpU6Pk3fal5EQ47EbvkvKeaFGylnb3NWw,606
 chellow/e/bill_importer.py,sha256=7UcnqNlKbJc2GhW9gy8sDp9GuqambJVpZLvbafOZztA,7411
@@ -27,7 +27,7 @@ chellow/e/elexon.py,sha256=ALhXS9Es7PV0z9ukPbIramn3cf3iLyFi-PMWPSm5iOs,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
 chellow/e/hh_importer.py,sha256=adfogJgPWsJurPOVSBLLQ7m68s1LhfQmDYpAFnGm0eM,21510
 chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
-chellow/e/hh_parser_df2.py,sha256=tRAoVUUoJDlfPopm6usEBnhJz7dXMc2_KEWbkW9Gyq4,4330
+chellow/e/hh_parser_df2.py,sha256=nZT198EZWo_G26IfJZncI_kOo-UdsXzFYDmDf3qyEs8,5089
 chellow/e/hh_parser_edf_csv.py,sha256=CLkkL1Z6BPgVV_3uwS6McmtMzgXkoEIoJnH8FsQk1C8,2839
 chellow/e/hh_parser_schneider_csv.py,sha256=m8FHwXp1Tbas91RPS7TriCskSNRuSaFf1SD9v2HXfSk,2366
 chellow/e/hh_parser_schneider_xlsx.py,sha256=Vtq0TNz-oojoKJm4PeH4ZwBp2I-mjArB9-FL71ctCHs,3590
@@ -58,7 +58,7 @@ chellow/e/bill_parsers/gdf_csv.py,sha256=ZfK3Oc6oP28p_P9DIevLNB_zW2WLcEJ3Lvb1gL3
 chellow/e/bill_parsers/haven_csv.py,sha256=0uENq8IgVNqdxfBQMBxLTSZWCOuDHXZC0xzk52SbfyE,13652
 chellow/e/bill_parsers/haven_edi.py,sha256=YGPHRxPOhje9s32jqPHHELni2tooOYj3cMC_qaZVPq4,16107
 chellow/e/bill_parsers/haven_edi_tprs.py,sha256=ZVX9CCqUybsot_Z0BEOJPvl9x5kSr7fEWyuJXvZDcz4,11841
-chellow/e/bill_parsers/mm.py,sha256=zeNF5F8uduQeEFjYOm5lfMS-DO3JjTB7oarA3H_Si5s,10019
+chellow/e/bill_parsers/mm.py,sha256=jFy5KlUnqfO0dsGgLMCEYAw0LPBIMBa9NDa-5ob2GwM,10033
 chellow/e/bill_parsers/nonsettlement_dc_stark_xlsx.py,sha256=yogXTuQHGRL7IiqvRWr2C9V24ez1j9Yx0128UygPE_k,4723
 chellow/e/bill_parsers/settlement_dc_stark_xlsx.py,sha256=PlEqCZuJ9DfQXeeYQ64jtf3ML7sUt_tt61QOOTnkE5c,6380
 chellow/e/bill_parsers/sse_edi.py,sha256=L85DOfNkqexeEIEr8pCBn_2sHJI-zEaw6cogpE3YyYM,15204
@@ -121,7 +121,7 @@ chellow/templates/chain.html,sha256=iOnSoDQOO_-1tfcxUHQKDAUnV4QbCYu5PPNmS4RkOg4,
 chellow/templates/csv_sites_monthly_duration.html,sha256=59gErG6KwA57z-qh2EJsVGvTUazqm85CSXefSfqPWcQ,486
 chellow/templates/downloads.html,sha256=R9QPcFz-PLJOX7rDlmquIk-Hp9Iq-thzWCTfOexSpXg,937
 chellow/templates/edi_viewer.html,sha256=szUthgHOdph6Of-7f_1LeC_zYlNJaMeS5ctn6xTAeiM,1437
-chellow/templates/general_import.html,sha256=SMemHGu466Be3_cAWSS6Hn-Kv-1psbEMDD6NQG6bfbI,1013
+chellow/templates/general_import.html,sha256=9ezzieDjaPBZ0nUJkMkzoDxWVzYtr4D-Dr2UCA5xV8U,1370
 chellow/templates/general_imports.html,sha256=VRmBqYsXsirm880WiFrJ3CknAXAoLR9Mnpz9cKjIeM8,13069
 chellow/templates/home.html,sha256=illtSabepcScYwmKss0o2gwvjXGBtr6LkfwmeL6ezSA,5655
 chellow/templates/input_date.html,sha256=rpgB5n0LfN8Y5djN_ZiuSxqdskxzCoKrEqI7hyJkVQo,1248
@@ -202,7 +202,7 @@ chellow/templates/e/dc_contract_edit.html,sha256=IObmbHQmeZ_LSpnYgabmhoSNUR3aPmm
 chellow/templates/e/dc_contract_hh_import.html,sha256=JncR3L6cOK4jghsGyr-itEqlIemXBXt3kL09wpqnQSE,856
 chellow/templates/e/dc_contract_hh_imports.html,sha256=eXFDGyzSgag4JRism81_p5yTzQOjCIXaVkQ8tl3dDcM,8172
 chellow/templates/e/dc_contract_properties.html,sha256=2EmA91gYWZcTOHQ3PxXQrpTOb8dm0OGjwRWsW6qNI9s,455
-chellow/templates/e/dc_contract_properties_edit.html,sha256=TGYcHTlWbk5WmjZPkdbHhRVZcOpEfhowCp2penZcGiE,1704
+chellow/templates/e/dc_contract_properties_edit.html,sha256=YWmG0dnzD6hf2kAFWCAjQWkHDPVRDi0jaUdLSLw-tfw,1901
 chellow/templates/e/dc_contracts.html,sha256=FPZUM7fkw-R6sRELHZCQTUkKSRqad8HOLHnyyYhQEEw,1819
 chellow/templates/e/dc_contracts_add.html,sha256=2lrGrNqAoKp16OiMoNDNlJxBMW3QdSPfgEFheSg826s,710
 chellow/templates/e/dc_rate_script.html,sha256=gfKUHV2IMsTQT-cYfjl4aiEBqGEWSj3uW22UZpoyGq4,1219
@@ -377,6 +377,6 @@ chellow/templates/g/supply_note_edit.html,sha256=b8mB6_ucBwoljp03iy6AgVaZUhGw3-1
 chellow/templates/g/supply_notes.html,sha256=6epNmZ3NKdXZz27fvmRUGeffg_oc1kmwuBeyRzQe3Rg,854
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1731581146.0.0.dist-info/METADATA,sha256=nVYIrIIfi0Wd791y4tLcCZoJMu9CQO31xsJCB77Pqfc,12204
-chellow-1731581146.0.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-chellow-1731581146.0.0.dist-info/RECORD,,
+chellow-1733390302.0.0.dist-info/METADATA,sha256=gaYBOwq3UhXxTCphDMWhGc0rFvubI80XXISsocq3EK0,12204
+chellow-1733390302.0.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+chellow-1733390302.0.0.dist-info/RECORD,,