chellow 1731581146.0.0__tar.gz → 1733390302.0.0__tar.gz

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1731581146.0.0
+Version: 1733390302.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1733390302.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1733390302.0.0"

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/chellow/e/bill_parsers/mm.py

@@ -147,6 +147,7 @@ READ_TYPE_LOOKUP = {
     " ": "E",
     "E": "E",
     "N": "N",
+    "S": "N",
 }
 
 

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/chellow/e/hh_parser_df2.py

@@ -4,7 +4,12 @@ from codecs import iterdecode
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import HH, parse_mpan_core, utc_datetime
+from chellow.utils import HH, hh_format, parse_mpan_core, utc_datetime
+
+
+def _append_new_message(messages, message):
+    if len(messages) == 0 or messages[-1] != message:
+        messages.append(message)
 
 
 def create_parser(reader, mpan_map, messages):
@@ -41,14 +46,6 @@ class StarkDf2HhParser:
                 if lline.startswith("#O"):
                     outstation_parts = lline[2:].split("-")
                     self.core = parse_mpan_core(outstation_parts[0])
-                    if self.core in self.mpan_map and self.mpan_map[self.core] is None:
-                        msg = f"The MPAN core {self.core} has been ignored"
-                        if len(self.messages) == 0 or self.messages[-1] != msg:
-                            self.messages.append(
-                                f"The MPAN core {self.core} has been ignored"
-                            )
-                        self.core = None
-                        continue
 
                 elif lline.startswith("#S"):
                     sensor = int(lline[2:].strip())
@@ -87,11 +84,31 @@ class StarkDf2HhParser:
                         raise BadRequest("Problem parsing the value: " + fields[2])
                     status = fields[3][-1]
 
-                    if self.core is None:
-                        continue
+                    mpan_core = self.core
+                    if self.core in self.mpan_map:
+                        mapping = self.mpan_map[self.core]
+                        from_date = mapping["from"]
+                        to_date = mapping["to"]
+                        if from_date <= start_date <= to_date:
+                            mpan_core = mapping["mpan"]
+
+                            if mpan_core is None:
+                                _append_new_message(
+                                    self.messages,
+                                    f"The MPAN core {self.core} is ignored from "
+                                    f"{hh_format(from_date)} to {hh_format(to_date)}",
+                                )
+                                continue
+                            else:
+                                _append_new_message(
+                                    self.messages,
+                                    f"The MPAN core {self.core} is changed to "
+                                    f"{mpan_core} from {hh_format(from_date)} to "
+                                    f"{hh_format(to_date)}",
+                                )
 
                     local_datum = {
-                        "mpan_core": self.core,
+                        "mpan_core": mpan_core,
                         "channel_type": self.channel_type,
                         "start_date": start_date,
                         "value": value,

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/chellow/general_import.py

@@ -1704,21 +1704,43 @@ for k in tuple(globals().keys()):
         typ_funcs[k[len(PREFIX) :]] = globals()[k]
 
 
+def _process_line(sess, hh_data, line, args):
+
+    action = add_arg(args, "action", line, 0).lower()
+    if action not in ALLOWED_ACTIONS:
+        raise BadRequest(f"The 'action' field must be one of {ALLOWED_ACTIONS}")
+    typ = add_arg(args, "type", line, 1).lower()
+    vals = line[2:]
+    if typ == "hh_datum":
+        if action == "insert":
+            hh_data.append(
+                {
+                    "mpan_core": parse_mpan_core(add_arg(args, "MPAN Core", vals, 0)),
+                    "start_date": parse_hh_start(add_arg(args, "Start Date", vals, 1)),
+                    "channel_type": parse_channel_type(
+                        add_arg(args, "Channel Type", vals, 2)
+                    ),
+                    "value": Decimal(add_arg(args, "Value", vals, 3)),
+                    "status": add_arg(args, "Status", vals, 4),
+                }
+            )
+    else:
+        try:
+            typ_func = typ_funcs[typ]
+            typ_func(sess, action, vals, args)
+        except KeyError:
+            raise BadRequest(f"The type {typ} is not recognized.")
+
+
 class GeneralImporter(threading.Thread):
     def __init__(self, f):
         threading.Thread.__init__(self)
         self.line_number = None
         self.f = f
-        self.rd_lock = threading.Lock()
         self.error_message = None
         self.args = []
-        self.hh_data = []
-
-    def get_fields(self):
-        fields = {"line_number": self.line_number, "error_message": self.error_message}
-        if self.error_message is not None:
-            fields["csv_line"] = self.args
-        return fields
+        self.failed_args = []
+        self.failed_lines = []
 
     def run(self):
         try:
@@ -1735,63 +1757,32 @@ class GeneralImporter(threading.Thread):
                     if len(line) > 0 and line[0].startswith("#"):
                         continue
 
-                    action = add_arg(self.args, "action", line, 0).lower()
-                    if action not in ALLOWED_ACTIONS:
-                        raise BadRequest(
-                            f"The 'action' field must be one of {ALLOWED_ACTIONS}"
-                        )
-                    typ = add_arg(self.args, "type", line, 1).lower()
-                    vals = line[2:]
-                    if typ == "hh_datum":
-                        if action == "insert":
-                            hh_data.append(
-                                {
-                                    "mpan_core": parse_mpan_core(
-                                        add_arg(self.args, "MPAN Core", vals, 0)
-                                    ),
-                                    "start_date": parse_hh_start(
-                                        add_arg(self.args, "Start Date", vals, 1)
-                                    ),
-                                    "channel_type": parse_channel_type(
-                                        add_arg(self.args, "Channel Type", vals, 2)
-                                    ),
-                                    "value": Decimal(
-                                        add_arg(self.args, "Value", vals, 3)
-                                    ),
-                                    "status": add_arg(self.args, "Status", vals, 4),
-                                }
-                            )
-                    else:
-                        try:
-                            typ_func = typ_funcs[typ]
-                            typ_func(sess, action, vals, self.args)
-                        except KeyError:
-                            raise BadRequest(f"The type {typ} is not recognized.")
+                    try:
+                        with sess.begin_nested():
+                            _process_line(sess, hh_data, line, self.args)
+
+                    except BadRequest as e:
+                        self.args.insert(0, ("Error", e.description))
+                        self.failed_args.append(self.args)
+                        self.failed_lines.append(line)
 
                 HhDatum.insert(sess, hh_data)
-                sess.commit()
+
+                if len(self.failed_lines) == 0:
+                    sess.commit()
+                else:
+                    sess.rollback()
         except BadRequest as e:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = e.description
-            finally:
-                self.rd_lock.release()
+            self.error_message = e.description
         except BaseException:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = traceback.format_exc()
-            finally:
-                self.rd_lock.release()
+            self.error_message = traceback.format_exc()
 
 
 def start_process(f):
-    try:
+    with process_lock:
         global process_id
-        process_lock.acquire()
         proc_id = process_id
         process_id += 1
-    finally:
-        process_lock.release()
 
     process = GeneralImporter(f)
     processes[proc_id] = process
@@ -1800,16 +1791,9 @@ def start_process(f):
 
 
 def get_process_ids():
-    try:
-        process_lock.acquire()
+    with process_lock:
         return processes.keys()
-    finally:
-        process_lock.release()
 
 
 def get_process(id):
-    try:
-        process_lock.acquire()
-        return processes[id]
-    finally:
-        process_lock.release()
+    return processes[id]

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/chellow/templates/e/dc_contract_properties_edit.html

@@ -39,8 +39,17 @@
   "private_key": "private_key", /* Remove if using password */
   "directories": ["downloads1", "downloads2"],
   "mpan_map": {  /* Optional */
-    "99 0993 2821 985": null,  /* Ignore MPAN */
-  },
+		"99 0993 2821 985": {
+			"mpan": null,  /* Ignore MPAN */
+			"from": 2024-12-04T10:00:00Z,
+			"to": 2025-01-12:00:00Z,
+		},
+		"99 0873 1942 952": {
+			"mpan": "99 7821 8541 389",
+			"from": 2024-11-16T22:00:00Z,
+			"to": 2025-02-18:00:00Z,
+		},
+	},
 }
 		</pre>
 	</code>

chellow-1733390302.0.0/chellow/templates/general_import.html

@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» General Imports » {{import_id}}
+{% endblock %}
+
+{% block nav %}
+	<a href="/general_imports">General Imports</a> &raquo;
+	{{import_id}}
+{% endblock %}
+
+{% block content %}
+	{% if importer.line_number %}
+		<p>Line number {{importer.line_number}}.</p>
+	{% endif %}
+
+	{% if importer.is_alive() %}
+			<p>Still running. Refresh the page to see latest progress.</p>
+	{% else %}
+		{% if importer.failed_args|length > 0 %}
+				<p>
+					The import has failed and so the whole import has been rolled back.
+				</p>
+
+				<h3>Failed Lines</h3>
+
+				{% for line in importer.failed_args %}
+
+					<table>
+						<tbody>
+							<tr>
+								{% for title, value in line %}
+									<th>{{title}}</th>
+								{% endfor %}
+							</tr>
+							<tr>
+								{% for title, value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						</tbody>
+					</table>
+				{% endfor %}
+
+				<h3>Failed Input</h3>
+
+				<table>
+					<tbody>
+						{% for line in importer.failed_lines %}
+							<tr>
+								{% for value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+			{% elif importer.error_message %}
+				<p>{{importer.error_message}}</p>
+			{% else %}
+				<p>The file has been imported successfully.</p>
+			{% endif %}
+	{% endif %}
+{% endblock %}

{chellow-1731581146.0.0 → chellow-1733390302.0.0}/chellow/views.py

@@ -755,14 +755,13 @@ def general_imports_post():
 @home.route("/general_imports/<int:import_id>")
 def general_import_get(import_id):
     try:
-        proc = chellow.general_import.get_process(import_id)
-        fields = proc.get_fields()
-        fields["is_alive"] = proc.is_alive()
-        fields["process_id"] = import_id
-        return render_template("general_import.html", **fields)
+        importer = chellow.general_import.get_process(import_id)
+        return render_template(
+            "general_import.html", import_id=import_id, importer=importer
+        )
     except BadRequest as e:
         flash(e.description)
-        return render_template("general_import.html", process_id=import_id)
+        return render_template("general_import.html", import_id=import_id)
 
 
 @home.route("/edi_viewer")

chellow-1731581146.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1731581146.0.0"

chellow-1731581146.0.0/chellow/templates/general_import.html

@@ -1,47 +0,0 @@
-{% extends "base.html" %}
-
-{% block title %}
-	» General Imports » {{process_id}}
-{% endblock %}
-
-{% block nav %}
-	<a href="/general_imports">General Imports</a> &raquo;
-	{{process_id}}
-{% endblock %}
-
-{% block content %}
-	{% if line_number %}
-		<p>Line number {{line_number}}.</p>
-	{% endif %}
-
-	{% if is_alive %}
-			<p>Still running. Refresh the page to see latest progress.</p>
-	{% else %}
-			{% if error_message %}
-					<p>
-						The import has failed and so the whole import has been rolled
-						back. The reason for the failure is:</p>
-					<pre>{{error_message}}</pre>
-			{% else %}
-					<p>The file has been imported successfully.</p>
-			{% endif %}
-	{% endif %}
-
-	{% if csv_line %}
-		<table>
-			<caption>Failed Line</caption>
-			<tbody>
-				<tr>
-					{% for title, value in csv_line %}
-						<th>{{title}}</th>
-					{% endfor %}
-				</tr>
-				<tr>
-					{% for title, value in csv_line %}
-						<td>{{value}}</td>
-					{% endfor %}
-				</tr>
-			</tbody>
-		</table>
-	{% endif %}
-{% endblock %}