chellow 1731435800.0.0__py3-none-any.whl → 1732281664.0.0__py3-none-any.whl

chellow/general_import.py

@@ -149,11 +149,13 @@ def general_import_era(sess, action, vals, args):
         comm_code = add_arg(args, "Comms Type", vals, 12)
         comm = era.comm if comm_code == NO_CHANGE else Comm.get_by_code(sess, comm_code)
 
-        ssc_code = add_arg(args, "SSC", vals, 13)
-        if ssc_code == NO_CHANGE:
-            ssc_code = era.ssc.code
-        elif len(ssc_code) == 0:
+        ssc_code_str = add_arg(args, "SSC", vals, 13)
+        if ssc_code_str == NO_CHANGE:
+            ssc_code = None if era.ssc is None else era.ssc.code
+        elif len(ssc_code_str) == 0:
             ssc_code = None
+        else:
+            ssc_code = ssc_code_str
 
         es_code = add_arg(args, "Energisation Status", vals, 14)
         if es_code == NO_CHANGE:
@@ -1702,21 +1704,43 @@ for k in tuple(globals().keys()):
         typ_funcs[k[len(PREFIX) :]] = globals()[k]
 
 
+def _process_line(sess, hh_data, line, args):
+
+    action = add_arg(args, "action", line, 0).lower()
+    if action not in ALLOWED_ACTIONS:
+        raise BadRequest(f"The 'action' field must be one of {ALLOWED_ACTIONS}")
+    typ = add_arg(args, "type", line, 1).lower()
+    vals = line[2:]
+    if typ == "hh_datum":
+        if action == "insert":
+            hh_data.append(
+                {
+                    "mpan_core": parse_mpan_core(add_arg(args, "MPAN Core", vals, 0)),
+                    "start_date": parse_hh_start(add_arg(args, "Start Date", vals, 1)),
+                    "channel_type": parse_channel_type(
+                        add_arg(args, "Channel Type", vals, 2)
+                    ),
+                    "value": Decimal(add_arg(args, "Value", vals, 3)),
+                    "status": add_arg(args, "Status", vals, 4),
+                }
+            )
+    else:
+        try:
+            typ_func = typ_funcs[typ]
+            typ_func(sess, action, vals, args)
+        except KeyError:
+            raise BadRequest(f"The type {typ} is not recognized.")
+
+
 class GeneralImporter(threading.Thread):
     def __init__(self, f):
         threading.Thread.__init__(self)
         self.line_number = None
         self.f = f
-        self.rd_lock = threading.Lock()
         self.error_message = None
         self.args = []
-        self.hh_data = []
-
-    def get_fields(self):
-        fields = {"line_number": self.line_number, "error_message": self.error_message}
-        if self.error_message is not None:
-            fields["csv_line"] = self.args
-        return fields
+        self.failed_args = []
+        self.failed_lines = []
 
     def run(self):
         try:
@@ -1733,63 +1757,32 @@ class GeneralImporter(threading.Thread):
                     if len(line) > 0 and line[0].startswith("#"):
                         continue
 
-                    action = add_arg(self.args, "action", line, 0).lower()
-                    if action not in ALLOWED_ACTIONS:
-                        raise BadRequest(
-                            f"The 'action' field must be one of {ALLOWED_ACTIONS}"
-                        )
-                    typ = add_arg(self.args, "type", line, 1).lower()
-                    vals = line[2:]
-                    if typ == "hh_datum":
-                        if action == "insert":
-                            hh_data.append(
-                                {
-                                    "mpan_core": parse_mpan_core(
-                                        add_arg(self.args, "MPAN Core", vals, 0)
-                                    ),
-                                    "start_date": parse_hh_start(
-                                        add_arg(self.args, "Start Date", vals, 1)
-                                    ),
-                                    "channel_type": parse_channel_type(
-                                        add_arg(self.args, "Channel Type", vals, 2)
-                                    ),
-                                    "value": Decimal(
-                                        add_arg(self.args, "Value", vals, 3)
-                                    ),
-                                    "status": add_arg(self.args, "Status", vals, 4),
-                                }
-                            )
-                    else:
-                        try:
-                            typ_func = typ_funcs[typ]
-                            typ_func(sess, action, vals, self.args)
-                        except KeyError:
-                            raise BadRequest(f"The type {typ} is not recognized.")
+                    try:
+                        with sess.begin_nested():
+                            _process_line(sess, hh_data, line, self.args)
+
+                    except BadRequest as e:
+                        self.args.insert(0, ("Error", e.description))
+                        self.failed_args.append(self.args)
+                        self.failed_lines.append(line)
 
                 HhDatum.insert(sess, hh_data)
-                sess.commit()
+
+                if len(self.failed_lines) == 0:
+                    sess.commit()
+                else:
+                    sess.rollback()
         except BadRequest as e:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = e.description
-            finally:
-                self.rd_lock.release()
+            self.error_message = e.description
         except BaseException:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = traceback.format_exc()
-            finally:
-                self.rd_lock.release()
+            self.error_message = traceback.format_exc()
 
 
 def start_process(f):
-    try:
+    with process_lock:
         global process_id
-        process_lock.acquire()
         proc_id = process_id
         process_id += 1
-    finally:
-        process_lock.release()
 
     process = GeneralImporter(f)
     processes[proc_id] = process
@@ -1798,16 +1791,9 @@ def start_process(f):
 
 
 def get_process_ids():
-    try:
-        process_lock.acquire()
+    with process_lock:
         return processes.keys()
-    finally:
-        process_lock.release()
 
 
 def get_process(id):
-    try:
-        process_lock.acquire()
-        return processes[id]
-    finally:
-        process_lock.release()
+    return processes[id]

chellow/reports/report_111.py

@@ -178,6 +178,7 @@ def content(
                 "bill-kwh",
                 "bill-net-gbp",
                 "bill-vat-gbp",
+                "bill-gross-gbp",
                 "bill-start-date",
                 "bill-finish-date",
                 "imp-mpan-core",
@@ -371,7 +372,13 @@ def _process_supply(
 
         covered_start = bill_start
         covered_finish = bill_start
-        covered_bdown = {"sum-msp-kwh": 0, "net-gbp": 0, "vat-gbp": 0, "problem": ""}
+        covered_bdown = {
+            "sum-msp-kwh": 0,
+            "net-gbp": 0,
+            "vat-gbp": 0,
+            "gross-gbp": 0,
+            "problem": "",
+        }
 
         vb_elems = set()
         enlarged = True
@@ -439,9 +446,23 @@ def _process_supply(
         primary_covered_bill = None
         for covered_bill in covered_bills.values():
             bill_ids.discard(covered_bill.id)
-            covered_bdown["net-gbp"] += float(covered_bill.net)
-            covered_bdown["vat-gbp"] += float(covered_bill.vat)
             covered_bdown["sum-msp-kwh"] += float(covered_bill.kwh)
+            for elem, val in (
+                ("net", covered_bill.net),
+                ("vat", covered_bill.vat),
+                ("gross", covered_bill.gross),
+            ):
+                covered_bdown[f"{elem}-gbp"] += float(val)
+                covered_elems.add(elem)
+                add_gap(
+                    caches,
+                    gaps,
+                    elem,
+                    covered_bill.start_date,
+                    covered_bill.finish_date,
+                    False,
+                    val,
+                )
             for k, v in loads(covered_bill.breakdown).items():
                 if k in ("raw_lines", "raw-lines", "vat"):
                     continue
@@ -599,7 +620,7 @@ def _process_supply(
 
             for dt, bl in vb_hhs.items():
                 for k, v in bl.items():
-                    if all((k.endswith("-gbp"), k != "net-gbp", v != 0)):
+                    if k.endswith("-gbp") and v != 0:
                         add_gap(caches, gaps, k[:-4], dt, dt, True, v)
 
             for k in virtual_bill.keys():
@@ -625,13 +646,13 @@ def _process_supply(
                 f"virtual bill. "
             )
 
-        try:
-            del virtual_bill["net-gbp"]
-        except KeyError:
-            pass
-
+        virtual_bill.pop("net-gbp", None)
+        virtual_bill.pop("gross-gbp", None)
         virtual_bill["net-gbp"] = sum(
-            v for k, v in virtual_bill.items() if k.endswith("-gbp")
+            v for k, v in virtual_bill.items() if k.endswith("-gbp") and k != "vat-gbp"
+        )
+        virtual_bill["gross-gbp"] = virtual_bill["net-gbp"] + virtual_bill.get(
+            "vat-gbp", 0
         )
 
         era = supply.find_era_at(sess, bill_finish)
@@ -665,6 +686,7 @@ def _process_supply(
             "bill-kwh": bill.kwh,
             "bill-net-gbp": bill.net,
             "bill-vat-gbp": bill.vat,
+            "bill-gross-gbp": bill.gross,
             "bill-start-date": bill_start,
             "bill-finish-date": bill_finish,
             "imp-mpan-core": imp_mpan_core,

chellow/templates/general_import.html

@@ -1,47 +1,64 @@
 {% extends "base.html" %}
 
 {% block title %}
-	» General Imports » {{process_id}}
+	» General Imports » {{import_id}}
 {% endblock %}
 
 {% block nav %}
 	<a href="/general_imports">General Imports</a> &raquo;
-	{{process_id}}
+	{{import_id}}
 {% endblock %}
 
 {% block content %}
-	{% if line_number %}
-		<p>Line number {{line_number}}.</p>
+	{% if importer.line_number %}
+		<p>Line number {{importer.line_number}}.</p>
 	{% endif %}
 
-	{% if is_alive %}
+	{% if importer.is_alive() %}
 			<p>Still running. Refresh the page to see latest progress.</p>
 	{% else %}
-			{% if error_message %}
-					<p>
-						The import has failed and so the whole import has been rolled
-						back. The reason for the failure is:</p>
-					<pre>{{error_message}}</pre>
+		{% if importer.failed_args|length > 0 %}
+				<p>
+					The import has failed and so the whole import has been rolled back.
+				</p>
+
+				<h3>Failed Lines</h3>
+
+				{% for line in importer.failed_args %}
+
+					<table>
+						<tbody>
+							<tr>
+								{% for title, value in line %}
+									<th>{{title}}</th>
+								{% endfor %}
+							</tr>
+							<tr>
+								{% for title, value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						</tbody>
+					</table>
+				{% endfor %}
+
+				<h3>Failed Input</h3>
+
+				<table>
+					<tbody>
+						{% for line in importer.failed_lines %}
+							<tr>
+								{% for value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+			{% elif importer.error_message %}
+				<p>{{importer.error_message}}</p>
 			{% else %}
-					<p>The file has been imported successfully.</p>
+				<p>The file has been imported successfully.</p>
 			{% endif %}
 	{% endif %}
-
-	{% if csv_line %}
-		<table>
-			<caption>Failed Line</caption>
-			<tbody>
-				<tr>
-					{% for title, value in csv_line %}
-						<th>{{title}}</th>
-					{% endfor %}
-				</tr>
-				<tr>
-					{% for title, value in csv_line %}
-						<td>{{value}}</td>
-					{% endfor %}
-				</tr>
-			</tbody>
-		</table>
-	{% endif %}
 {% endblock %}

chellow/views.py

@@ -755,14 +755,13 @@ def general_imports_post():
 @home.route("/general_imports/<int:import_id>")
 def general_import_get(import_id):
     try:
-        proc = chellow.general_import.get_process(import_id)
-        fields = proc.get_fields()
-        fields["is_alive"] = proc.is_alive()
-        fields["process_id"] = import_id
-        return render_template("general_import.html", **fields)
+        importer = chellow.general_import.get_process(import_id)
+        return render_template(
+            "general_import.html", import_id=import_id, importer=importer
+        )
     except BadRequest as e:
         flash(e.description)
-        return render_template("general_import.html", process_id=import_id)
+        return render_template("general_import.html", import_id=import_id)
 
 
 @home.route("/edi_viewer")

{chellow-1731435800.0.0.dist-info → chellow-1732281664.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1731435800.0.0
+Version: 1732281664.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1731435800.0.0.dist-info → chellow-1732281664.0.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ chellow/bank_holidays.py,sha256=T_utYMwe_g1dz5X-aOTdIPryg49SvB7QsWM1yphlqG8,4423
 chellow/commands.py,sha256=ESBe9ZWj1c3vdZgqMZ9gFvYAB3hRag2R1PzOwuw9yFo,1302
 chellow/dloads.py,sha256=dixp-O0MF2_mlwrnKx3D9DH09Qu05BjTo0rZfigTjR4,5534
 chellow/edi_lib.py,sha256=alu20x9ZX06iPfnNI9dEJzuP6RIf4We3Y_M_bl7RrcY,51789
-chellow/general_import.py,sha256=Ia2Y40NEwqOYBHUwafJRcI34EO0WMX6FSBQJtGHeY0M,65874
+chellow/general_import.py,sha256=tcYZK2N7HOFJSkYyy3MMxVBEvMFjfB9pjF0ksujtkFA,65113
 chellow/models.py,sha256=9LLROzS6gXZGICk0ITXhCyTtKaIY7HW4bMqWg4Mc-xw,244221
 chellow/national_grid.py,sha256=czwIZqzJndSGhEMQ5YzI6hRBhvjkM6VRVYXybf4_KXg,4377
 chellow/proxy.py,sha256=cVXIktPlX3tQ1BYcwxq0nJXKE6r3DtFTtfFHPq55HaM,1351
@@ -12,7 +12,7 @@ chellow/rate_server.py,sha256=fg-Pf_9Hk3bXmC9riPQNGQxBvLvBa_WtNYdwDCjnCSg,5678
 chellow/rrun.py,sha256=1Kt2q_K9UoDG_nsZz-Q6XJiMNKroWqlqFdxn2M6Q8CA,2088
 chellow/testing.py,sha256=Od4HHH6pZrhJ_De118_F55RJEKmAvhUH2S24QE9qFQk,3635
 chellow/utils.py,sha256=32qPWEGzCPKPhSM7ztpzcR6ZG74sVZanScDIdK50Rq4,19308
-chellow/views.py,sha256=eb7WUqqu7LmAX5m_9ICJFVpo0i8WZ9GV89AkqRA3aac,84383
+chellow/views.py,sha256=qJrvhVjb3KL_sHDQl5dLhi0aqUe_IUYhrjiPjdk4lQI,84317
 chellow/e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/aahedc.py,sha256=d2usudp7KYWpU6Pk3fal5EQ47EbvkvKeaFGylnb3NWw,606
 chellow/e/bill_importer.py,sha256=7UcnqNlKbJc2GhW9gy8sDp9GuqambJVpZLvbafOZztA,7411
@@ -75,7 +75,7 @@ chellow/gas/transportation.py,sha256=Bkg8TWOs-v0ES-4qqwbleiOhqbE_t2KauUx9JYMZELM
 chellow/gas/views.py,sha256=R4Ky8syioWSQJLTuSKVLsc266eDv8a6WEo_e60u_4ug,59021
 chellow/reports/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/reports/report_109.py,sha256=FmRWG8wQC97cj0nqQH7mrfbYesRIKWXQLh6ERHvtHrU,11212
-chellow/reports/report_111.py,sha256=rgwoOPfvbfwIC_IjBqs2e8umF_YTtpyejLrX_uCgyHE,28145
+chellow/reports/report_111.py,sha256=MBXMABwHonjH2cvkqFYC0Vx7rU7LB2mJxXsEjI0RvTU,28828
 chellow/reports/report_169.py,sha256=tnTiHmhigcgZ7E8rUSfgzsVYUCW947xBZZq04-KqfK8,9287
 chellow/reports/report_181.py,sha256=mCL6vXfrj8r0kL-LilDS0SnSu5tJOzCq7eMJVx9zpEw,4999
 chellow/reports/report_183.py,sha256=DZX-hHJPl_Tbgkt31C_cuLZg_5L2b6iCPQ5foOZizR0,8817
@@ -121,7 +121,7 @@ chellow/templates/chain.html,sha256=iOnSoDQOO_-1tfcxUHQKDAUnV4QbCYu5PPNmS4RkOg4,
 chellow/templates/csv_sites_monthly_duration.html,sha256=59gErG6KwA57z-qh2EJsVGvTUazqm85CSXefSfqPWcQ,486
 chellow/templates/downloads.html,sha256=R9QPcFz-PLJOX7rDlmquIk-Hp9Iq-thzWCTfOexSpXg,937
 chellow/templates/edi_viewer.html,sha256=szUthgHOdph6Of-7f_1LeC_zYlNJaMeS5ctn6xTAeiM,1437
-chellow/templates/general_import.html,sha256=SMemHGu466Be3_cAWSS6Hn-Kv-1psbEMDD6NQG6bfbI,1013
+chellow/templates/general_import.html,sha256=9ezzieDjaPBZ0nUJkMkzoDxWVzYtr4D-Dr2UCA5xV8U,1370
 chellow/templates/general_imports.html,sha256=VRmBqYsXsirm880WiFrJ3CknAXAoLR9Mnpz9cKjIeM8,13069
 chellow/templates/home.html,sha256=illtSabepcScYwmKss0o2gwvjXGBtr6LkfwmeL6ezSA,5655
 chellow/templates/input_date.html,sha256=rpgB5n0LfN8Y5djN_ZiuSxqdskxzCoKrEqI7hyJkVQo,1248
@@ -377,6 +377,6 @@ chellow/templates/g/supply_note_edit.html,sha256=b8mB6_ucBwoljp03iy6AgVaZUhGw3-1
 chellow/templates/g/supply_notes.html,sha256=6epNmZ3NKdXZz27fvmRUGeffg_oc1kmwuBeyRzQe3Rg,854
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1731435800.0.0.dist-info/METADATA,sha256=ilPn6kcxryT74lMCin7HXAJqf48jbjfuyV1Y3qmf9dA,12204
-chellow-1731435800.0.0.dist-info/WHEEL,sha256=WJ9WQ4-pUYxfD_tEj5GvKSG9KSULNumqkTQdolV8mME,87
-chellow-1731435800.0.0.dist-info/RECORD,,
+chellow-1732281664.0.0.dist-info/METADATA,sha256=6kGXaB73s2rZ4c_deiOhKUWc_n1lJjFA6vIJa_ToVYE,12204
+chellow-1732281664.0.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+chellow-1732281664.0.0.dist-info/RECORD,,

{chellow-1731435800.0.0.dist-info → chellow-1732281664.0.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.26.2
+Generator: hatchling 1.26.3
 Root-Is-Purelib: true
 Tag: py3-none-any