chellow 1731435800.0.0__tar.gz → 1732281664.0.0__tar.gz

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1731435800.0.0
+Version: 1732281664.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1732281664.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1732281664.0.0"

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/chellow/general_import.py

@@ -149,11 +149,13 @@ def general_import_era(sess, action, vals, args):
         comm_code = add_arg(args, "Comms Type", vals, 12)
         comm = era.comm if comm_code == NO_CHANGE else Comm.get_by_code(sess, comm_code)
 
-        ssc_code = add_arg(args, "SSC", vals, 13)
-        if ssc_code == NO_CHANGE:
-            ssc_code = era.ssc.code
-        elif len(ssc_code) == 0:
+        ssc_code_str = add_arg(args, "SSC", vals, 13)
+        if ssc_code_str == NO_CHANGE:
+            ssc_code = None if era.ssc is None else era.ssc.code
+        elif len(ssc_code_str) == 0:
             ssc_code = None
+        else:
+            ssc_code = ssc_code_str
 
         es_code = add_arg(args, "Energisation Status", vals, 14)
         if es_code == NO_CHANGE:
@@ -1702,21 +1704,43 @@ for k in tuple(globals().keys()):
         typ_funcs[k[len(PREFIX) :]] = globals()[k]
 
 
+def _process_line(sess, hh_data, line, args):
+
+    action = add_arg(args, "action", line, 0).lower()
+    if action not in ALLOWED_ACTIONS:
+        raise BadRequest(f"The 'action' field must be one of {ALLOWED_ACTIONS}")
+    typ = add_arg(args, "type", line, 1).lower()
+    vals = line[2:]
+    if typ == "hh_datum":
+        if action == "insert":
+            hh_data.append(
+                {
+                    "mpan_core": parse_mpan_core(add_arg(args, "MPAN Core", vals, 0)),
+                    "start_date": parse_hh_start(add_arg(args, "Start Date", vals, 1)),
+                    "channel_type": parse_channel_type(
+                        add_arg(args, "Channel Type", vals, 2)
+                    ),
+                    "value": Decimal(add_arg(args, "Value", vals, 3)),
+                    "status": add_arg(args, "Status", vals, 4),
+                }
+            )
+    else:
+        try:
+            typ_func = typ_funcs[typ]
+            typ_func(sess, action, vals, args)
+        except KeyError:
+            raise BadRequest(f"The type {typ} is not recognized.")
+
+
 class GeneralImporter(threading.Thread):
     def __init__(self, f):
         threading.Thread.__init__(self)
         self.line_number = None
         self.f = f
-        self.rd_lock = threading.Lock()
         self.error_message = None
         self.args = []
-        self.hh_data = []
-
-    def get_fields(self):
-        fields = {"line_number": self.line_number, "error_message": self.error_message}
-        if self.error_message is not None:
-            fields["csv_line"] = self.args
-        return fields
+        self.failed_args = []
+        self.failed_lines = []
 
     def run(self):
         try:
@@ -1733,63 +1757,32 @@ class GeneralImporter(threading.Thread):
                     if len(line) > 0 and line[0].startswith("#"):
                         continue
 
-                    action = add_arg(self.args, "action", line, 0).lower()
-                    if action not in ALLOWED_ACTIONS:
-                        raise BadRequest(
-                            f"The 'action' field must be one of {ALLOWED_ACTIONS}"
-                        )
-                    typ = add_arg(self.args, "type", line, 1).lower()
-                    vals = line[2:]
-                    if typ == "hh_datum":
-                        if action == "insert":
-                            hh_data.append(
-                                {
-                                    "mpan_core": parse_mpan_core(
-                                        add_arg(self.args, "MPAN Core", vals, 0)
-                                    ),
-                                    "start_date": parse_hh_start(
-                                        add_arg(self.args, "Start Date", vals, 1)
-                                    ),
-                                    "channel_type": parse_channel_type(
-                                        add_arg(self.args, "Channel Type", vals, 2)
-                                    ),
-                                    "value": Decimal(
-                                        add_arg(self.args, "Value", vals, 3)
-                                    ),
-                                    "status": add_arg(self.args, "Status", vals, 4),
-                                }
-                            )
-                    else:
-                        try:
-                            typ_func = typ_funcs[typ]
-                            typ_func(sess, action, vals, self.args)
-                        except KeyError:
-                            raise BadRequest(f"The type {typ} is not recognized.")
+                    try:
+                        with sess.begin_nested():
+                            _process_line(sess, hh_data, line, self.args)
+
+                    except BadRequest as e:
+                        self.args.insert(0, ("Error", e.description))
+                        self.failed_args.append(self.args)
+                        self.failed_lines.append(line)
 
                 HhDatum.insert(sess, hh_data)
-                sess.commit()
+
+                if len(self.failed_lines) == 0:
+                    sess.commit()
+                else:
+                    sess.rollback()
         except BadRequest as e:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = e.description
-            finally:
-                self.rd_lock.release()
+            self.error_message = e.description
         except BaseException:
-            try:
-                self.rd_lock.acquire()
-                self.error_message = traceback.format_exc()
-            finally:
-                self.rd_lock.release()
+            self.error_message = traceback.format_exc()
 
 
 def start_process(f):
-    try:
+    with process_lock:
         global process_id
-        process_lock.acquire()
         proc_id = process_id
         process_id += 1
-    finally:
-        process_lock.release()
 
     process = GeneralImporter(f)
     processes[proc_id] = process
@@ -1798,16 +1791,9 @@ def start_process(f):
 
 
 def get_process_ids():
-    try:
-        process_lock.acquire()
+    with process_lock:
         return processes.keys()
-    finally:
-        process_lock.release()
 
 
 def get_process(id):
-    try:
-        process_lock.acquire()
-        return processes[id]
-    finally:
-        process_lock.release()
+    return processes[id]

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/chellow/reports/report_111.py

@@ -178,6 +178,7 @@ def content(
                 "bill-kwh",
                 "bill-net-gbp",
                 "bill-vat-gbp",
+                "bill-gross-gbp",
                 "bill-start-date",
                 "bill-finish-date",
                 "imp-mpan-core",
@@ -371,7 +372,13 @@ def _process_supply(
 
         covered_start = bill_start
         covered_finish = bill_start
-        covered_bdown = {"sum-msp-kwh": 0, "net-gbp": 0, "vat-gbp": 0, "problem": ""}
+        covered_bdown = {
+            "sum-msp-kwh": 0,
+            "net-gbp": 0,
+            "vat-gbp": 0,
+            "gross-gbp": 0,
+            "problem": "",
+        }
 
         vb_elems = set()
         enlarged = True
@@ -439,9 +446,23 @@ def _process_supply(
         primary_covered_bill = None
         for covered_bill in covered_bills.values():
             bill_ids.discard(covered_bill.id)
-            covered_bdown["net-gbp"] += float(covered_bill.net)
-            covered_bdown["vat-gbp"] += float(covered_bill.vat)
             covered_bdown["sum-msp-kwh"] += float(covered_bill.kwh)
+            for elem, val in (
+                ("net", covered_bill.net),
+                ("vat", covered_bill.vat),
+                ("gross", covered_bill.gross),
+            ):
+                covered_bdown[f"{elem}-gbp"] += float(val)
+                covered_elems.add(elem)
+                add_gap(
+                    caches,
+                    gaps,
+                    elem,
+                    covered_bill.start_date,
+                    covered_bill.finish_date,
+                    False,
+                    val,
+                )
             for k, v in loads(covered_bill.breakdown).items():
                 if k in ("raw_lines", "raw-lines", "vat"):
                     continue
@@ -599,7 +620,7 @@ def _process_supply(
 
             for dt, bl in vb_hhs.items():
                 for k, v in bl.items():
-                    if all((k.endswith("-gbp"), k != "net-gbp", v != 0)):
+                    if k.endswith("-gbp") and v != 0:
                         add_gap(caches, gaps, k[:-4], dt, dt, True, v)
 
             for k in virtual_bill.keys():
@@ -625,13 +646,13 @@ def _process_supply(
                 f"virtual bill. "
             )
 
-        try:
-            del virtual_bill["net-gbp"]
-        except KeyError:
-            pass
-
+        virtual_bill.pop("net-gbp", None)
+        virtual_bill.pop("gross-gbp", None)
         virtual_bill["net-gbp"] = sum(
-            v for k, v in virtual_bill.items() if k.endswith("-gbp")
+            v for k, v in virtual_bill.items() if k.endswith("-gbp") and k != "vat-gbp"
+        )
+        virtual_bill["gross-gbp"] = virtual_bill["net-gbp"] + virtual_bill.get(
+            "vat-gbp", 0
         )
 
         era = supply.find_era_at(sess, bill_finish)
@@ -665,6 +686,7 @@ def _process_supply(
             "bill-kwh": bill.kwh,
             "bill-net-gbp": bill.net,
             "bill-vat-gbp": bill.vat,
+            "bill-gross-gbp": bill.gross,
             "bill-start-date": bill_start,
             "bill-finish-date": bill_finish,
             "imp-mpan-core": imp_mpan_core,

chellow-1732281664.0.0/chellow/templates/general_import.html

@@ -0,0 +1,64 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» General Imports » {{import_id}}
+{% endblock %}
+
+{% block nav %}
+	<a href="/general_imports">General Imports</a> &raquo;
+	{{import_id}}
+{% endblock %}
+
+{% block content %}
+	{% if importer.line_number %}
+		<p>Line number {{importer.line_number}}.</p>
+	{% endif %}
+
+	{% if importer.is_alive() %}
+			<p>Still running. Refresh the page to see latest progress.</p>
+	{% else %}
+		{% if importer.failed_args|length > 0 %}
+				<p>
+					The import has failed and so the whole import has been rolled back.
+				</p>
+
+				<h3>Failed Lines</h3>
+
+				{% for line in importer.failed_args %}
+
+					<table>
+						<tbody>
+							<tr>
+								{% for title, value in line %}
+									<th>{{title}}</th>
+								{% endfor %}
+							</tr>
+							<tr>
+								{% for title, value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						</tbody>
+					</table>
+				{% endfor %}
+
+				<h3>Failed Input</h3>
+
+				<table>
+					<tbody>
+						{% for line in importer.failed_lines %}
+							<tr>
+								{% for value in line %}
+									<td>{{value}}</td>
+								{% endfor %}
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+			{% elif importer.error_message %}
+				<p>{{importer.error_message}}</p>
+			{% else %}
+				<p>The file has been imported successfully.</p>
+			{% endif %}
+	{% endif %}
+{% endblock %}

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/chellow/views.py

@@ -755,14 +755,13 @@ def general_imports_post():
 @home.route("/general_imports/<int:import_id>")
 def general_import_get(import_id):
     try:
-        proc = chellow.general_import.get_process(import_id)
-        fields = proc.get_fields()
-        fields["is_alive"] = proc.is_alive()
-        fields["process_id"] = import_id
-        return render_template("general_import.html", **fields)
+        importer = chellow.general_import.get_process(import_id)
+        return render_template(
+            "general_import.html", import_id=import_id, importer=importer
+        )
     except BadRequest as e:
         flash(e.description)
-        return render_template("general_import.html", process_id=import_id)
+        return render_template("general_import.html", import_id=import_id)
 
 
 @home.route("/edi_viewer")

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/test/reports/test_report_111.py

@@ -327,9 +327,11 @@ def virtual_bill(ds):
     expected = (
         "a b,hjk,N,10.00,10.00,10.00,2009-07-10 01:00,2009-07-10 01:00,"
         "22 7867 6232 781,,CI017,Water Works,2009-07-10 01:00,2009-07-10 01:00,"
-        "1,0,,,,,,,0,10.0,0,10.0,10.0,,0,,,0,10.0,,0.1,0.1,,,0,,,"
+        "1,0,,,,,,,0,10.0,0,10.0,10.0,0,10.0,10.0,0,10.0,10.0,,0.1,0.1,,,0,,,"
         "virtual-off-rate,0.1\r\n"
     )
+    print(expected)
+    print(f.getvalue())
     assert f.getvalue() == expected
 
 

{chellow-1731435800.0.0 → chellow-1732281664.0.0}/test/test_general_import.py

@@ -768,7 +768,129 @@ def test_general_import_era_insert_nhh(sess):
     general_import_era(sess, action, vals, args)
 
 
-def test_general_import_era_update(sess):
+def test_general_import_era_update_hh(sess):
+    vf = to_utc(ct_datetime(1996, 1, 1))
+    site = Site.insert(sess, "CI017", "Water Works")
+    market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    market_role_X = MarketRole.insert(sess, "X", "Supplier")
+    market_role_M = MarketRole.insert(sess, "M", "Mop")
+    market_role_C = MarketRole.insert(sess, "C", "HH Dc")
+    market_role_R = MarketRole.insert(sess, "R", "Distributor")
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+    participant.insert_party(sess, market_role_X, "Fusion Ltc", vf, None, None)
+    participant.insert_party(sess, market_role_C, "Fusion DC", vf, None, None)
+    mop_contract = Contract.insert_mop(
+        sess, "Fusion", participant, "", {}, vf, None, {}
+    )
+    dc_contract = Contract.insert_dc(
+        sess, "Fusion DC 2000", participant, "", {}, vf, None, {}
+    )
+    pc = Pc.insert(sess, "00", "HH", vf, None)
+    insert_cops(sess)
+    cop = Cop.get_by_code(sess, "5")
+    insert_comms(sess)
+    comm = Comm.get_by_code(sess, "GSM")
+    exp_supplier_contract = Contract.insert_supplier(
+        sess, "Fusion Supplier 2000", participant, "", {}, vf, None, {}
+    )
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    meter_type = MeterType.insert(sess, "C5", "COP 1-5", vf, None)
+    meter_payment_type = MeterPaymentType.insert(sess, "CR", "Credit", vf, None)
+    mtc = Mtc.insert(sess, "845", False, True, vf, None)
+    mtc_participant = MtcParticipant.insert(
+        sess,
+        mtc,
+        participant,
+        "HH COP5 And Above With Comms",
+        False,
+        True,
+        meter_type,
+        meter_payment_type,
+        0,
+        vf,
+        None,
+    )
+    insert_voltage_levels(sess)
+    voltage_level = VoltageLevel.get_by_code(sess, "HV")
+    llfc = dno.insert_llfc(
+        sess, "521", "Export (HV)", voltage_level, False, False, vf, None
+    )
+    MtcLlfc.insert(sess, mtc_participant, llfc, vf, None)
+    insert_sources(sess)
+    source = Source.get_by_code(sess, "grid")
+    insert_energisation_statuses(sess)
+    energisation_status = EnergisationStatus.get_by_code(sess, "E")
+    gsp_group = GspGroup.insert(sess, "_L", "South Western")
+    insert_dtc_meter_types(sess)
+    dtc_meter_type = DtcMeterType.get_by_code(sess, "H")
+    site.insert_e_supply(
+        sess,
+        source,
+        None,
+        "Bob",
+        utc_datetime(2000, 1, 1),
+        None,
+        gsp_group,
+        mop_contract,
+        "773",
+        dc_contract,
+        "ghyy3",
+        "hgjeyhuw",
+        dno,
+        pc,
+        "845",
+        cop,
+        comm,
+        None,
+        energisation_status,
+        dtc_meter_type,
+        None,
+        None,
+        None,
+        None,
+        None,
+        "22 7867 6232 781",
+        "521",
+        exp_supplier_contract,
+        "7748",
+        361,
+    )
+    action = "update"
+    vals = [
+        "2278676232781",
+        "2019-09-08 00:00",
+        "{no change}",
+        "{no change}",
+        "Fusion",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+        "{no change}",
+    ]
+    args = []
+    general_import_era(sess, action, vals, args)
+
+
+def test_general_import_era_update_nhh(sess):
     vf = to_utc(ct_datetime(1996, 1, 1))
     site = Site.insert(sess, "CI017", "Water Works")
     market_role_Z = MarketRole.insert(sess, "Z", "Non-core")

chellow-1731435800.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1731435800.0.0"

chellow-1731435800.0.0/chellow/templates/general_import.html

@@ -1,47 +0,0 @@
-{% extends "base.html" %}
-
-{% block title %}
-	» General Imports » {{process_id}}
-{% endblock %}
-
-{% block nav %}
-	<a href="/general_imports">General Imports</a> &raquo;
-	{{process_id}}
-{% endblock %}
-
-{% block content %}
-	{% if line_number %}
-		<p>Line number {{line_number}}.</p>
-	{% endif %}
-
-	{% if is_alive %}
-			<p>Still running. Refresh the page to see latest progress.</p>
-	{% else %}
-			{% if error_message %}
-					<p>
-						The import has failed and so the whole import has been rolled
-						back. The reason for the failure is:</p>
-					<pre>{{error_message}}</pre>
-			{% else %}
-					<p>The file has been imported successfully.</p>
-			{% endif %}
-	{% endif %}
-
-	{% if csv_line %}
-		<table>
-			<caption>Failed Line</caption>
-			<tbody>
-				<tr>
-					{% for title, value in csv_line %}
-						<th>{{title}}</th>
-					{% endfor %}
-				</tr>
-				<tr>
-					{% for title, value in csv_line %}
-						<td>{{value}}</td>
-					{% endfor %}
-				</tr>
-			</tbody>
-		</table>
-	{% endif %}
-{% endblock %}