chellow 1716366171.0.0__tar.gz → 1716452334.0.0__tar.gz

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1716366171.0.0
+Version: 1716452334.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
-Requires-Dist: odio==0.0.22
+Requires-Dist: odio==0.0.23
 Requires-Dist: openpyxl==3.1.2
 Requires-Dist: paramiko==3.4.0
 Requires-Dist: pep3143daemon==0.0.6
@@ -20,8 +20,8 @@ Requires-Dist: pympler==1.0.1
 Requires-Dist: pypdf==3.17.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
-Requires-Dist: requests==2.31.0
-Requires-Dist: sqlalchemy==2.0.28
+Requires-Dist: requests==2.32.2
+Requires-Dist: sqlalchemy==2.0.30
 Requires-Dist: waitress==2.1.2
 Requires-Dist: xlrd==2.0.1
 Requires-Dist: zish==0.1.11

chellow-1716452334.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1716452334.0.0"

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/chellow/models.py

@@ -6400,8 +6400,23 @@ class ReportRun(Base, PersistentClass):
     def insert(sess, name, user, title, data):
         report_run = ReportRun(name, user, title, data)
         sess.add(report_run)
+        sess.flush()
         return report_run
 
+    @staticmethod
+    def w_update(report_run_id, state):
+        with Session() as wsess:
+            report_run = ReportRun.get_by_id(wsess, report_run_id)
+            report_run.update(state)
+            wsess.commit()
+
+    @staticmethod
+    def w_insert_row(report_run_id, tab, titles, values, properties):
+        with Session() as wsess:
+            report_run = ReportRun.get_by_id(wsess, report_run_id)
+            report_run.insert_row(wsess, tab, titles, values, properties)
+            wsess.commit()
+
 
 def _jsonize(val):
     if isinstance(val, dict):

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/chellow/reports/report_111.py

@@ -29,9 +29,9 @@ from chellow.models import (
     Llfc,
     MarketRole,
     MtcParticipant,
+    RSession,
     RegisterRead,
     ReportRun,
-    Session,
     Site,
     SiteEra,
     Supply,
@@ -102,19 +102,17 @@ def content(
     report_run_id,
 ):
     caches = {}
-    tmp_file = sess = supply_id = report_run = None
+    tmp_file = sess = supply_id = None
     forecast_date = to_utc(Datetime.max)
 
     try:
-        with Session() as sess:
+        with RSession() as sess:
             user = User.get_by_id(sess, user_id)
             tmp_file = open_file(
                 f"bill_check_{fname_additional}.csv", user, mode="w", newline=""
             )
             writer = csv.writer(tmp_file, lineterminator="\n")
 
-            report_run = ReportRun.get_by_id(sess, report_run_id)
-
             bills = (
                 sess.query(Bill)
                 .order_by(Bill.supply_id, Bill.reference)
@@ -216,30 +214,26 @@ def content(
                     virtual_bill_titles,
                     writer,
                     titles,
-                    report_run,
+                    report_run_id,
                 )
-
-            report_run.update("finished")
-            sess.commit()
+        ReportRun.w_update(report_run_id, "finished")
 
     except BadRequest as e:
-        if report_run is not None:
-            report_run.update("problem")
         if supply_id is None:
             prefix = "Problem: "
         else:
             prefix = f"Problem with supply {supply_id}:"
         tmp_file.write(prefix + e.description)
+        ReportRun.w_update(report_run_id, "problem")
     except BaseException:
-        if report_run is not None:
-            report_run.update("interrupted")
+        msg = traceback.format_exc()
+        sys.stderr.write(msg + "\n")
         if supply_id is None:
             prefix = "Problem: "
         else:
             prefix = f"Problem with supply {supply_id}:"
-        msg = traceback.format_exc()
-        sys.stderr.write(msg + "\n")
         tmp_file.write(prefix + msg)
+        ReportRun.w_update(report_run_id, "interrupted")
     finally:
         if tmp_file is not None:
             tmp_file.close()
@@ -318,7 +312,7 @@ def _process_supply(
     virtual_bill_titles,
     writer,
     titles,
-    report_run,
+    report_run_id,
 ):
     gaps = {}
     data_sources = {}
@@ -761,8 +755,8 @@ def _process_supply(
         values["batch_id"] = bill.batch.id
         values["supply_id"] = supply.id
         values["site_id"] = None if site_code is None else site.id
-        report_run.insert_row(
-            sess, "", report_run_titles, values, {"is_checked": False}
+        ReportRun.w_insert_row(
+            report_run_id, "", report_run_titles, values, {"is_checked": False}
         )
 
         for bill in sess.query(Bill).filter(
@@ -783,7 +777,7 @@ def _process_supply(
                     )
 
         # Avoid long-running transactions
-        sess.commit()
+        sess.rollback()
 
     clumps = []
     for element, elgap in sorted(gaps.items()):
@@ -837,7 +831,7 @@ def _process_supply(
         vals["supply_id"] = supply.id
         vals["site_id"] = None if site_code is None else site.id
 
-        report_run.insert_row(sess, "", titles, vals, {"is_checked": False})
+        ReportRun.w_insert_row(report_run_id, "", titles, vals, {"is_checked": False})
 
     # Avoid long-running transactions
-    sess.commit()
+    sess.rollback()

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/chellow/reports/report_387.py

@@ -10,7 +10,7 @@ from sqlalchemy.sql.expression import null, true
 
 from chellow.dloads import open_file
 from chellow.e.computer import SupplySource, forecast_date
-from chellow.models import Era, Session, Site, SiteEra, Supply
+from chellow.models import Era, RSession, Site, SiteEra, Supply
 from chellow.utils import (
     csv_make_val,
     hh_format,
@@ -25,7 +25,7 @@ def content(supply_id, start_date, finish_date, user):
     caches = {}
     f = None
     try:
-        with Session() as sess:
+        with RSession() as sess:
             supply = Supply.get_by_id(sess, supply_id)
 
             f_date = forecast_date()

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/chellow/reports/report_59.py

@@ -486,7 +486,45 @@ def _process_site(
                     .all()
                 )
                 if len(bills) > 0:
-                    month_data = {}
+                    imp_supplier_contract = last_era.imp_supplier_contract
+                    imp_supplier_contract_name = (
+                        None
+                        if imp_supplier_contract is None
+                        else imp_supplier_contract.name
+                    )
+                    exp_supplier_contract = last_era.exp_supplier_contract
+                    exp_supplier_contract_name = (
+                        None
+                        if exp_supplier_contract is None
+                        else exp_supplier_contract.name
+                    )
+                    era_associates = {
+                        s.site.code for s in last_era.site_eras if not s.is_physical
+                    }
+                    vals = {
+                        "creation-date": now,
+                        "start-date": chunk_start,
+                        "finish-date": chunk_finish,
+                        "imp-mpan-core": last_era.imp_mpan_core,
+                        "exp-mpan-core": last_era.exp_mpan_core,
+                        "site-code": site.code,
+                        "site-name": site.name,
+                        "associated-site-codes": era_associates,
+                        "era-start-date": None,
+                        "era-finish-date": None,
+                        "imp-supplier-contract": imp_supplier_contract_name,
+                        "imp-non-actual-hhs": None,
+                        "imp-non-actual-kwh": None,
+                        "exp-supplier-contract": exp_supplier_contract_name,
+                        "exp-non-actual-hhs": None,
+                        "exp-non-actual-kwh": None,
+                        "metering-type": last_era.meter_category,
+                        "source": last_era.supply.source.code,
+                        "generator-type": None,
+                        "supply-name": last_era.supply.name,
+                        "msn": last_era.msn,
+                        "pc": last_era.pc.code,
+                    }
                     for name in (
                         "import-net",
                         "export-net",
@@ -500,10 +538,10 @@ def _process_site(
                         "billed-import-net",
                     ):
                         for sname in ("kwh", "gbp"):
-                            month_data[name + "-" + sname] = 0
-                    month_data["billed-supplier-import-net-gbp"] = 0
-                    month_data["billed-dc-import-net-gbp"] = 0
-                    month_data["billed-mop-import-net-gbp"] = 0
+                            vals[name + "-" + sname] = 0
+                    vals["billed-supplier-import-net-gbp"] = 0
+                    vals["billed-dc-import-net-gbp"] = 0
+                    vals["billed-mop-import-net-gbp"] = 0
 
                     for bill in bills:
                         bill_role_code = bill.batch.contract.market_role.code
@@ -528,42 +566,12 @@ def _process_site(
                         else:
                             raise BadRequest("Role code not recognized.")
 
-                        for data in month_data, site_data:
+                        for data in vals, site_data:
                             data["billed-import-net-kwh"] += bill_prop_kwh
                             data["billed-import-net-gbp"] += bill_prop_gbp
                             data[key] += bill_prop_gbp
 
-                    imp_supplier_contract = last_era.imp_supplier_contract
-                    exp_supplier_contract = last_era.exp_supplier_contract
-                    out = [
-                        now,
-                        last_era.imp_mpan_core,
-                        (
-                            None
-                            if imp_supplier_contract is None
-                            else imp_supplier_contract.name
-                        ),
-                        last_era.exp_mpan_core,
-                        (
-                            None
-                            if exp_supplier_contract is None
-                            else exp_supplier_contract.name
-                        ),
-                        chunk_start,
-                        last_era.meter_category,
-                        last_era.supply.source.code,
-                        None,
-                        last_era.supply.name,
-                        last_era.msn,
-                        last_era.pc.code,
-                        site.code,
-                        site.name,
-                        None,
-                        start_date,
-                        finish_date,
-                    ] + [month_data[t] for t in summary_titles]
-
-                    era_rows.append([make_val(v) for v in out])
+                    era_rows.append([make_val(vals.get(t)) for t in era_titles])
         first_era = (
             sess.execute(
                 select(Era).where(Era.supply == supply).order_by(Era.start_date)

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/pyproject.toml

@@ -14,16 +14,16 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-	"odio==0.0.22",
+	"odio==0.0.23",
 	"pypdf==3.17.0",
 	"pg8000==1.31.1",
 	"Flask==2.3.3",
-	"SQLAlchemy==2.0.28",
+	"SQLAlchemy==2.0.30",
 	"flask-restx==1.2.0",
 	"openpyxl==3.1.2",
 	"python-dateutil==2.8.2",
 	"pytz==2022.6",
-	"requests==2.31.0",
+	"requests==2.32.2",
 	"waitress==2.1.2",
 	"pep3143daemon==0.0.6",
 	"pip>=9.0.1",

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/test/reports/test_report_111.py

@@ -102,14 +102,12 @@ def test_http_supplier_batch_with_mpan_cores(mocker, client, sess):
 
 
 def test_process_supply(sess):
-    valid_from = to_utc(ct_datetime(1996, 1, 1))
+    vf = to_utc(ct_datetime(1996, 1, 1))
     site = Site.insert(sess, "CI017", "Water Works")
 
     market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
     participant = Participant.insert(sess, "CALB", "AK Industries")
-    participant.insert_party(
-        sess, market_role_Z, "None core", utc_datetime(2000, 1, 1), None, None
-    )
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
     bank_holiday_rate_script = {"bank_holidays": []}
     Contract.insert_non_core(
         sess,
@@ -124,22 +122,16 @@ def test_process_supply(sess):
     market_role_M = MarketRole.insert(sess, "M", "Mop")
     market_role_C = MarketRole.insert(sess, "C", "HH Dc")
     market_role_R = MarketRole.insert(sess, "R", "Distributor")
-    participant.insert_party(
-        sess, market_role_M, "Fusion Mop Ltd", utc_datetime(2000, 1, 1), None, None
-    )
-    participant.insert_party(
-        sess, market_role_X, "Fusion Ltc", utc_datetime(2000, 1, 1), None, None
-    )
-    participant.insert_party(
-        sess, market_role_C, "Fusion DC", utc_datetime(2000, 1, 1), None, None
-    )
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+    participant.insert_party(sess, market_role_X, "Fusion Ltc", vf, None, None)
+    participant.insert_party(sess, market_role_C, "Fusion DC", vf, None, None)
     mop_contract = Contract.insert_mop(
-        sess, "Fusion", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
+        sess, "Fusion", participant, "", {}, vf, None, {}
     )
     dc_contract = Contract.insert_dc(
-        sess, "Fusion DC 2000", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
+        sess, "Fusion DC 2000", participant, "", {}, vf, None, {}
     )
-    pc = Pc.insert(sess, "00", "hh", utc_datetime(2000, 1, 1), None)
+    pc = Pc.insert(sess, "00", "hh", vf, None)
     insert_cops(sess)
     cop = Cop.get_by_code(sess, "5")
     insert_comms(sess)
@@ -174,19 +166,8 @@ def virtual_bill(ds):
         None,
         {},
     )
-    dno = participant.insert_party(
-        sess, market_role_R, "WPD", utc_datetime(2000, 1, 1), None, "22"
-    )
-    Contract.insert_dno(
-        sess,
-        dno.dno_code,
-        participant,
-        "",
-        {},
-        valid_from,
-        None,
-        {},
-    )
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    Contract.insert_dno(sess, dno.dno_code, participant, "", {}, vf, None, {})
     meter_type = MeterType.insert(sess, "C5", "COP 1-5", utc_datetime(2000, 1, 1), None)
     meter_payment_type = MeterPaymentType.insert(
         sess, "CR", "Credit", utc_datetime(1996, 1, 1), None
@@ -224,7 +205,7 @@ def virtual_bill(ds):
         utc_datetime(1996, 1, 1),
         None,
     )
-    MtcLlfc.insert(sess, mtc_participant, llfc, valid_from, None)
+    MtcLlfc.insert(sess, mtc_participant, llfc, vf, None)
     insert_sources(sess)
     source = Source.get_by_code(sess, "net")
     insert_energisation_statuses(sess)
@@ -283,6 +264,7 @@ def virtual_bill(ds):
         supply,
     )
     report_run = ReportRun.insert(sess, "bill_check", None, "", {})
+    report_run_id = report_run.id
     sess.commit()
 
     report_context = {}
@@ -340,7 +322,7 @@ def virtual_bill(ds):
         virtual_bill_titles,
         writer,
         titles,
-        report_run,
+        report_run_id,
     )
     expected = (
         "a b,hjk,N,10.00,10.00,10.00,2009-07-10 01:00,2009-07-10 01:00,"
@@ -352,21 +334,19 @@ def virtual_bill(ds):
 
 
 def test_content(sess, mocker):
-    valid_from = to_utc(ct_datetime(1996, 1, 1))
+    vf = to_utc(ct_datetime(1996, 1, 1))
     site = Site.insert(sess, "CI017", "Water Works")
 
     market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
     participant = Participant.insert(sess, "CALB", "AK Industries")
-    participant.insert_party(
-        sess, market_role_Z, "None core", utc_datetime(2000, 1, 1), None, None
-    )
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
     bank_holiday_rate_script = {"bank_holidays": []}
     Contract.insert_non_core(
         sess,
         "bank_holidays",
         "",
         {},
-        utc_datetime(2000, 1, 1),
+        vf,
         None,
         bank_holiday_rate_script,
     )
@@ -374,22 +354,16 @@ def test_content(sess, mocker):
     market_role_M = MarketRole.insert(sess, "M", "Mop")
     market_role_C = MarketRole.insert(sess, "C", "HH Dc")
     market_role_R = MarketRole.insert(sess, "R", "Distributor")
-    participant.insert_party(
-        sess, market_role_M, "Fusion Mop Ltd", utc_datetime(2000, 1, 1), None, None
-    )
-    participant.insert_party(
-        sess, market_role_X, "Fusion Ltc", utc_datetime(2000, 1, 1), None, None
-    )
-    participant.insert_party(
-        sess, market_role_C, "Fusion DC", utc_datetime(2000, 1, 1), None, None
-    )
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+    participant.insert_party(sess, market_role_X, "Fusion Ltc", vf, None, None)
+    participant.insert_party(sess, market_role_C, "Fusion DC", vf, None, None)
     mop_contract = Contract.insert_mop(
-        sess, "Fusion", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
+        sess, "Fusion", participant, "", {}, vf, None, {}
     )
     dc_contract = Contract.insert_dc(
-        sess, "Fusion DC 2000", participant, "", {}, utc_datetime(2000, 1, 1), None, {}
+        sess, "Fusion DC 2000", participant, "", {}, vf, None, {}
     )
-    pc = Pc.insert(sess, "00", "hh", utc_datetime(2000, 1, 1), None)
+    pc = Pc.insert(sess, "00", "hh", vf, None)
     insert_cops(sess)
     cop = Cop.get_by_code(sess, "5")
     insert_comms(sess)
@@ -424,31 +398,11 @@ def virtual_bill(ds):
         None,
         {},
     )
-    dno = participant.insert_party(
-        sess, market_role_R, "WPD", utc_datetime(2000, 1, 1), None, "22"
-    )
-    Contract.insert_dno(
-        sess,
-        dno.dno_code,
-        participant,
-        "",
-        {},
-        valid_from,
-        None,
-        {},
-    )
-    meter_type = MeterType.insert(sess, "C5", "COP 1-5", utc_datetime(2000, 1, 1), None)
-    meter_payment_type = MeterPaymentType.insert(
-        sess, "CR", "Credit", utc_datetime(1996, 1, 1), None
-    )
-    mtc = Mtc.insert(
-        sess,
-        "845",
-        False,
-        True,
-        utc_datetime(1996, 1, 1),
-        None,
-    )
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    Contract.insert_dno(sess, dno.dno_code, participant, "", {}, vf, None, {})
+    meter_type = MeterType.insert(sess, "C5", "COP 1-5", vf, None)
+    meter_payment_type = MeterPaymentType.insert(sess, "CR", "Credit", vf, None)
+    mtc = Mtc.insert(sess, "845", False, True, vf, None)
     mtc_participant = MtcParticipant.insert(
         sess,
         mtc,
@@ -459,22 +413,15 @@ def virtual_bill(ds):
         meter_type,
         meter_payment_type,
         0,
-        utc_datetime(1996, 1, 1),
+        vf,
         None,
     )
     insert_voltage_levels(sess)
     voltage_level = VoltageLevel.get_by_code(sess, "HV")
     llfc = dno.insert_llfc(
-        sess,
-        "510",
-        "PC 5-8 & HH HV",
-        voltage_level,
-        False,
-        True,
-        utc_datetime(1996, 1, 1),
-        None,
+        sess, "510", "PC 5-8 & HH HV", voltage_level, False, True, vf, None
     )
-    MtcLlfc.insert(sess, mtc_participant, llfc, valid_from, None)
+    MtcLlfc.insert(sess, mtc_participant, llfc, vf, None)
     insert_sources(sess)
     source = Source.get_by_code(sess, "net")
     insert_energisation_statuses(sess)
@@ -534,7 +481,12 @@ def virtual_bill(ds):
     )
     user_role = UserRole.insert(sess, "editor")
     user = User.insert(sess, "admin@example.com", "admin", user_role, None)
+    user_id = user.id
     report_run = ReportRun.insert(sess, "", user, "", {})
+    report_run_id = report_run.id
+    batch_id = batch.id
+    bill_id = bill.id
+    supplier_contract_id = supplier_contract.id
     sess.commit()
 
     virtual_bill_titles = [
@@ -578,13 +530,13 @@ def virtual_bill(ds):
     mocker.patch("chellow.reports.report_111.open_file", return_value=mock_file)
 
     content(
-        batch.id,
-        bill.id,
-        supplier_contract.id,
-        valid_from,
-        valid_from,
-        user.id,
+        batch_id,
+        bill_id,
+        supplier_contract_id,
+        vf,
+        vf,
+        user_id,
         [],
         "",
-        report_run.id,
+        report_run_id,
     )

{chellow-1716366171.0.0 → chellow-1716452334.0.0}/test/reports/test_report_387.py

@@ -218,6 +218,7 @@ def virtual_bill(ds):
         None,
         None,
     )
+    supply_id = supply.id
     insert_bill_types(sess)
     bill_type = sess.execute(select(BillType).where(BillType.code == "N")).scalar_one()
     batch.insert_bill(
@@ -245,10 +246,9 @@ def virtual_bill(ds):
     mock_file.close = mocker.Mock()
     mocker.patch("chellow.reports.report_387.open_file", return_value=mock_file)
 
-    content(supply.id, start_date, start_date, user)
+    content(supply_id, start_date, start_date, user)
 
     actual = mock_file.getvalue()
-    print(actual)
 
     expected = [
         [