chellow 1710934022.0.0__tar.gz → 1712589951.0.0__tar.gz

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1710934022.0.0
+Version: 1712589951.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1712589951.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1712589951.0.0"

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/__init__.py

@@ -63,8 +63,8 @@ def get_importer_modules():
     )
 
 
-def create_app(testing=False):
-    app = Flask("chellow", instance_relative_config=True)
+def create_app(testing=False, instance_path=None):
+    app = Flask("chellow", instance_relative_config=True, instance_path=instance_path)
     app.wsgi_app = MsProxy(app.wsgi_app)
     app.secret_key = os.urandom(24)
     start_sqlalchemy()
@@ -79,7 +79,6 @@ def create_app(testing=False):
 
     if not testing:
         db_upgrade(app.root_path)
-        chellow.dloads.startup(Path(app.instance_path))
         chellow.e.hh_importer.startup()
 
         with Session() as sess:
@@ -90,6 +89,8 @@ def create_app(testing=False):
             api_props = props.get("api", {})
             api.description = api_props.get("description", "Access Chellow data")
 
+    chellow.dloads.startup(Path(app.instance_path), run_deleter=(not testing))
+
     for module in get_importer_modules():
         if not testing:
             module.startup()

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/dloads.py

@@ -20,12 +20,14 @@ download_path = None
 SERIAL_DIGITS = 5
 
 
-def startup(instance_path):
+def startup(instance_path, run_deleter=True):
     global file_deleter
     global download_id
     global download_path
-    file_deleter = FileDeleter()
-    file_deleter.start()
+
+    if run_deleter:
+        file_deleter = FileDeleter()
+        file_deleter.start()
 
     download_path = instance_path / "downloads"
     download_path.mkdir(parents=True, exist_ok=True)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/e/views.py

@@ -1569,7 +1569,6 @@ def era_edit_form_get(era_id):
     try:
         era = Era.get_by_id(g.sess, era_id)
         ct_now = ct_datetime_now()
-        cops = g.sess.scalars(select(Cop).order_by(Cop.code))
         comms = g.sess.scalars(select(Comm).order_by(Comm.code))
         energisation_statuses = g.sess.scalars(
             select(EnergisationStatus).order_by(EnergisationStatus.code)
@@ -1781,6 +1780,11 @@ def era_edit_form_get(era_id):
         else:
             mtc_participant = None
 
+        if mtc_participant is None:
+            cops = g.sess.scalars(select(Cop).order_by(Cop.code))
+        else:
+            cops = Cop.get_valid(g.sess, mtc_participant.meter_type)
+
         if pc.code == "00":
             imp_llfcs_q = (
                 select(Llfc)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/models.py

@@ -510,6 +510,17 @@ class Cop(Base, PersistentClass):
             raise BadRequest(f"The CoP with code {code} can't be found.")
         return typ
 
+    @staticmethod
+    def get_valid(sess, meter_type):
+
+        if meter_type.code == "C5":
+            q = select(Cop).where(Cop.code.in_(["1", "2", "3", "4", "5"]))
+        elif meter_type.code in ["6A", "6B", "6C", "6D"]:
+            q = select(Cop).where(Cop.code == meter_type.code.lower())
+        else:
+            q = select(Cop)
+        return sess.scalars(q.order_by(Cop.code))
+
     __tablename__ = "cop"
     id = Column(Integer, primary_key=True)
     code = Column(String, unique=True, nullable=False)
@@ -3700,12 +3711,9 @@ class Era(Base, PersistentClass):
                             f"{hh_format(finish_date)}."
                         )
 
-        if (
-            self.mtc_participant.meter_type.code == "C5"
-            and cop.code not in ["1", "2", "3", "4", "5"]
-            or self.mtc_participant.meter_type.code in ["6A", "6B", "6C", "6D"]
-            and cop.code.upper() != self.mtc_participant.meter_type.code
-        ):
+        if cop.code not in [
+            c.code for c in Cop.get_valid(sess, self.mtc_participant.meter_type)
+        ]:
             raise BadRequest(
                 f"The CoP of {cop.code} is not compatible with the meter type code "
                 f"of {self.mtc_participant.meter_type.code}."

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_181.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 
@@ -10,11 +9,11 @@ from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
 import chellow.e.computer
 import chellow.e.duos
 import chellow.e.tnuos
-from chellow.models import Era, Pc, Session, Site, SiteEra, Source, Supply
+from chellow.dloads import open_file
+from chellow.models import Era, Pc, Session, Site, SiteEra, Source, Supply, User
 from chellow.utils import (
     HH,
     c_months_u,
@@ -144,28 +143,30 @@ def _write_sites(sess, caches, writer, year, site_id):
         sess.rollback()
 
 
-def content(year, site_id, user):
+def content(year, site_id, user_id):
     caches = {}
     f = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names("output.csv", user)
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("output.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             _write_sites(sess, caches, writer, year, site_id)
 
     except BadRequest as e:
         writer.writerow([e.description])
     except BaseException:
-        writer.writerow([traceback.format_exc()])
+        msg = traceback.format_exc()
+        print(msg)
+        if writer is not None:
+            writer.writerow([msg])
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
     site_id = req_int("site_id") if "site_id" in request.values else None
     year = req_int("year")
-    threading.Thread(target=content, args=(year, site_id, g.user)).start()
+    threading.Thread(target=content, args=(year, site_id, g.user.id)).start()
     return chellow_redirect("/downloads", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_241.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 from datetime import datetime as Datetime
@@ -14,8 +13,9 @@ from sqlalchemy.sql.expression import null, or_, true
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.computer
-from chellow.models import Era, Session, Site, SiteEra, Supply
+from chellow.dloads import open_file
+from chellow.e.computer import SupplySource, contract_func, forecast_date
+from chellow.models import Era, Session, Site, SiteEra, Supply, User
 from chellow.utils import HH, csv_make_val, hh_format, hh_max, hh_min, req_bool, req_int
 from chellow.views import chellow_redirect
 
@@ -29,15 +29,13 @@ def content(
     finish_day,
     is_import,
     supply_id,
-    user,
+    user_id,
 ):
     caches = {}
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                "daily_supplier_virtual_bill.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("daily_supplier_virtual_bill.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             start_date = Datetime(start_year, start_month, start_day, tzinfo=pytz.utc)
             finish_date = (
@@ -47,7 +45,7 @@ def content(
             )
 
             supply = Supply.get_by_id(sess, supply_id)
-            forecast_date = chellow.computer.forecast_date()
+            fd = forecast_date()
             day_start = start_date
             header_titles = [
                 "MPAN Core",
@@ -71,9 +69,7 @@ def content(
                 else:
                     cont = era.exp_supplier_contract
 
-                for title in chellow.computer.contract_func(
-                    caches, cont, "virtual_bill_titles"
-                )():
+                for title in contract_func(caches, cont, "virtual_bill_titles")():
                     if title not in bill_titles:
                         bill_titles.append(title)
 
@@ -94,11 +90,11 @@ def content(
                     chunk_start = hh_max(era.start_date, day_start)
                     chunk_finish = hh_min(era.finish_date, day_finish)
 
-                    ss = chellow.computer.SupplySource(
+                    ss = SupplySource(
                         sess,
                         chunk_start,
                         chunk_finish,
-                        forecast_date,
+                        fd,
                         era,
                         is_import,
                         caches,
@@ -120,9 +116,7 @@ def content(
                         ss.years_back > 0,
                     ]
 
-                    chellow.computer.contract_func(
-                        caches, ss.supplier_contract, "virtual_bill"
-                    )(ss)
+                    contract_func(caches, ss.supplier_contract, "virtual_bill")(ss)
                     bill = ss.supplier_bill
                     for title in bill_titles:
                         if title in bill:
@@ -144,7 +138,6 @@ def content(
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
@@ -159,18 +152,16 @@ def do_get(sess):
     is_import = req_bool("is_import")
     supply_id = req_int("supply_id")
 
-    threading.Thread(
-        target=content,
-        args=(
-            start_year,
-            start_month,
-            start_day,
-            finish_year,
-            finish_month,
-            finish_day,
-            is_import,
-            supply_id,
-            g.user,
-        ),
-    ).start()
+    args = (
+        start_year,
+        start_month,
+        start_day,
+        finish_year,
+        finish_month,
+        finish_day,
+        is_import,
+        supply_id,
+        g.user.id,
+    )
+    threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_asset_comparison.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -13,8 +12,8 @@ from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
-from chellow.models import Contract, Era, ReportRun, Session, Site, SiteEra
+from chellow.dloads import open_file
+from chellow.models import Contract, Era, ReportRun, Session, Site, SiteEra, User
 from chellow.views import chellow_redirect
 
 STATUSES_ACTIVE = ("IN USE / IN SERVICE", "STORED SPARE")
@@ -147,13 +146,12 @@ def _process_sites(sess, file_like, writer, props, report_run):
 FNAME = "asset_comparison"
 
 
-def content(user, file_like, report_run_id):
+def content(user_id, file_like, report_run_id):
+    f = report_run = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                f"{FNAME}.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file(f"{FNAME}.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             report_run = ReportRun.get_by_id(sess, report_run_id)
 
@@ -171,11 +169,11 @@ def content(user, file_like, report_run_id):
             report_run.insert_row(sess, "", ["error"], {"error": msg}, {})
             sess.commit()
         sys.stderr.write(msg)
-        writer.writerow([msg])
+        if writer is not None:
+            writer.writerow([msg])
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_post(sess):
@@ -194,6 +192,6 @@ def do_post(sess):
         },
     )
     sess.commit()
-    args = user, StringIO(file_item.read().decode("utf8")), report_run.id
+    args = user.id, StringIO(file_item.read().decode("utf8")), report_run.id
     threading.Thread(target=content, args=args).start()
     return chellow_redirect(f"/report_runs/{report_run.id}", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_csv_llfcs.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 
@@ -7,18 +6,18 @@ from flask import g
 
 from sqlalchemy.orm import joinedload
 
-import chellow.dloads
-from chellow.models import Llfc, Session
+from chellow.dloads import open_file
+from chellow.models import Llfc, Session, User
 from chellow.utils import hh_format
 from chellow.views import chellow_redirect
 
 
-def content(user):
+def content(user_id):
     f = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names("llfcs.csv", user)
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("llfcs.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             writer.writerow(
                 (
@@ -53,14 +52,16 @@ def content(user):
                     )
                 )
     except BaseException:
-        writer.writerow([traceback.format_exc()])
+        msg = traceback.format_exc()
+        print(msg)
+        if writer is not None:
+            writer.writerow([msg])
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
-    args = (g.user,)
+    args = (g.user.id,)
     threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_csv_site_snags.py

@@ -1,6 +1,4 @@
 import csv
-import os
-import sys
 import threading
 import traceback
 from datetime import datetime as Datetime, timedelta as Timedelta
@@ -12,20 +10,18 @@ import pytz
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import null
 
-import chellow.dloads
-from chellow.models import Session, Site, Snag
+from chellow.dloads import open_file
+from chellow.models import Session, Site, Snag, User
 from chellow.utils import hh_format
 from chellow.views import chellow_redirect
 
 
-def content(user):
+def content(user_id):
     f = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                "site_snags.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("site_snags.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             writer.writerow(
                 (
@@ -74,15 +70,15 @@ def content(user):
                 )
     except BaseException:
         msg = traceback.format_exc()
-        sys.stderr.write(msg)
-        writer.writerow([msg])
+        print(msg)
+        if writer is not None:
+            writer.writerow([msg])
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
-    args = (g.user,)
+    args = (g.user.id,)
     threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_g_virtual_bills.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -12,10 +11,10 @@ from sqlalchemy.sql.expression import null, true
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import contract_func, forecast_date
 from chellow.gas.engine import GDataSource
-from chellow.models import GContract, GEra, Session, Site, SiteGEra
+from chellow.models import GContract, GEra, Session, Site, SiteGEra, User
 from chellow.utils import (
     c_months_u,
     hh_format,
@@ -29,14 +28,13 @@ from chellow.utils import (
 from chellow.views import chellow_redirect
 
 
-def content(start_date, finish_date, g_contract_id, user):
+def content(start_date, finish_date, g_contract_id, user_id):
     report_context = {}
+    f = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                "gas_virtual_bills.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("gas_virtual_bills.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
 
             g_contract = GContract.get_by_id(sess, g_contract_id)
@@ -125,11 +123,11 @@ def content(start_date, finish_date, g_contract_id, user):
     except BaseException:
         msg = traceback.format_exc()
         sys.stderr.write(msg)
-        writer.writerow([msg])
+        if writer is not None:
+            writer.writerow([msg])
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
@@ -137,7 +135,6 @@ def do_get(sess):
     finish_date = req_date("finish")
     g_contract_id = req_int("g_contract_id")
 
-    threading.Thread(
-        target=content, args=(start_date, finish_date, g_contract_id, g.user)
-    ).start()
+    args = start_date, finish_date, g_contract_id, g.user.id
+    threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_sscs.py

@@ -1,4 +1,3 @@
-import os
 import sys
 import threading
 import traceback
@@ -13,7 +12,7 @@ from sqlalchemy.orm import joinedload
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import MeasurementRequirement, Session, Ssc, Tpr, User
 from chellow.utils import req_bool
 from chellow.views import chellow_redirect
@@ -54,9 +53,7 @@ def content(
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
 
-            running_name, finished_name = chellow.dloads.make_names("sscs.ods", user)
-
-            rf = open(running_name, "wb")
+            rf = open_file("sscs.ods", user, "wb")
 
             for ssc in sess.scalars(select(Ssc).order_by(Ssc.code)):
                 ssc_rows.append(
@@ -100,8 +97,7 @@ def content(
         ssc_rows.append(["Problem " + msg])
         if rf is None:
             msg = traceback.format_exc()
-            r_name, f_name = chellow.dloads.make_names("error.txt", None)
-            ef = open(r_name, "w")
+            ef = open_file("error.txt", None, "w")
             ef.write(msg + "\n")
             ef.close()
         else:
@@ -109,7 +105,6 @@ def content(
     finally:
         if rf is not None:
             rf.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/chellow/reports/report_supply_contacts.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -10,7 +9,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import null
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import (
     Contract,
     Era,
@@ -33,10 +32,7 @@ def content(user_id, report_run_id):
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(
-                f"{FNAME}.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            f = open_file(f"{FNAME}.csv", user, mode="w", newline="")
             report_run = ReportRun.get_by_id(sess, report_run_id)
 
             _process(sess, f, report_run)
@@ -56,7 +52,6 @@ def content(user_id, report_run_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def _process(sess, f, report_run):
@@ -129,13 +124,7 @@ def _process(sess, f, report_run):
 
 
 def do_get(sess):
-    report_run = ReportRun.insert(
-        sess,
-        FNAME,
-        g.user,
-        FNAME,
-        {},
-    )
+    report_run = ReportRun.insert(sess, FNAME, g.user, FNAME, {})
     sess.commit()
     threading.Thread(target=content, args=(g.user.id, report_run.id)).start()
     return chellow_redirect(f"/report_runs/{report_run.id}", 303)

{chellow-1710934022.0.0 → chellow-1712589951.0.0}/test/conftest.py

@@ -1,3 +1,5 @@
+from tempfile import TemporaryDirectory
+
 from flask.testing import FlaskClient
 
 from jinja2 import Environment, PackageLoader, select_autoescape
@@ -39,7 +41,8 @@ def fresh_db():
 def app(fresh_db):
     chellow.e.bill_importer.import_id = 0
     chellow.e.bill_importer.imports.clear()
-    return create_app(testing=True)
+    with TemporaryDirectory() as td:
+        yield create_app(testing=True, instance_path=td)
 
 
 @pytest.fixture

chellow-1712589951.0.0/test/reports/test_g_virtual_bills.py

@@ -0,0 +1,24 @@
+import chellow.dloads
+from chellow.models import (
+    GContract,
+    User,
+    UserRole,
+)
+from chellow.reports.report_g_virtual_bills import content
+from chellow.utils import ct_datetime, to_utc
+
+
+def test_content(sess):
+    editor = UserRole.insert(sess, "editor")
+    user = User.insert(sess, "admin@example.com", "xxx", editor, None)
+    user_id = user.id
+    vf = to_utc(ct_datetime(2020, 1, 1))
+    g_contract = GContract.insert(sess, False, "Fusion 2020", "", {}, vf, None, {})
+    sess.commit()
+
+    start_date = to_utc(ct_datetime(2020, 1, 1))
+    finish_date = to_utc(ct_datetime(2020, 1, 31, 23, 30))
+    content(start_date, finish_date, g_contract.id, user_id)
+
+    files = list(p.name for p in chellow.dloads.download_path.iterdir())
+    assert files == ["00000_FINISHED_adminexamplecom_gas_virtual_bills.csv"]