chellow 1706691842.0.0__tar.gz → 1707137154.0.0__tar.gz

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/.github/workflows/chellow.yml

@@ -24,7 +24,7 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -44,7 +44,7 @@ jobs:
       id-token: write
     if: github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -64,7 +64,7 @@ jobs:
       id-token: write
     if: github.ref == 'refs/heads/test'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1706691842.0.0
+Version: 1707137154.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1707137154.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1707137154.0.0"

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/__init__.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime as Datetime
 from importlib.metadata import version
+from pathlib import Path
 
 from flask import Flask, Response, g, make_response, render_template, request
 
@@ -78,7 +79,7 @@ def create_app(testing=False):
 
     if not testing:
         db_upgrade(app.root_path)
-        chellow.dloads.startup(app.instance_path)
+        chellow.dloads.startup(Path(app.instance_path))
         chellow.e.hh_importer.startup()
 
         with Session() as sess:

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/dloads.py

@@ -6,6 +6,10 @@ import threading
 import time
 import traceback
 
+from zipfile import ZIP_DEFLATED, ZipFile
+
+from werkzeug.exceptions import BadRequest
+
 from chellow.utils import utc_datetime_now
 
 
@@ -23,23 +27,54 @@ def startup(instance_path):
     file_deleter = FileDeleter()
     file_deleter.start()
 
-    download_path = os.path.join(instance_path, "downloads")
+    download_path = instance_path / "downloads"
+    download_path.mkdir(parents=True, exist_ok=True)
+
+    dirs = sorted(download_path.iterdir(), reverse=True)
+    if len(dirs) > 0:
+        download_id = int(dirs[0].name[:SERIAL_DIGITS]) + 1
+
+
+class DloadFile:
+    def __init__(self, running_name, finished_name, mode, newline, is_zip):
+        self.running_name = running_name
+        self.finished_name = finished_name
+        if is_zip:
+            self.f = ZipFile(running_name, mode=mode, compression=ZIP_DEFLATED)
+        else:
+            self.f = self.running_name.open(mode=mode, newline=newline)
+
+    def _check_exists(self):
+        if not self.running_name.exists():
+            raise BadRequest("Output file has been deleted.")
+
+    def write(self, b):
+        self._check_exists()
+        self.f.write(b)
+
+    def seek(self, offset, whence=0):
+        self._check_exists()
+        return self.f.seek(offset, whence)
+
+    def close(self):
+        self.f.close()
+        self._check_exists()
+        self.running_name.rename(self.finished_name)
 
-    if not os.path.exists(download_path):
-        os.makedirs(download_path)
+    def __enter__(self):
+        pass
 
-    files = sorted(os.listdir(download_path), reverse=True)
-    if len(files) > 0:
-        download_id = int(files[0][:SERIAL_DIGITS]) + 1
+    def __exit__(self):
+        self.close()
 
 
-def make_names(base, user):
+def open_file(base, user, mode="r", newline=None, is_zip=False):
     global download_id
 
     base = base.replace("/", "").replace(" ", "")
     try:
         lock.acquire()
-        if len(os.listdir(download_path)) == 0:
+        if len(list(download_path.iterdir())) == 0:
             download_id = 0
         serial = str(download_id).zfill(SERIAL_DIGITS)
         download_id += 1
@@ -56,8 +91,8 @@ def make_names(base, user):
         uname = un.replace("@", "").replace(".", "").replace("\\", "")
 
     names = tuple("_".join((serial, v, uname, base)) for v in ("RUNNING", "FINISHED"))
-    print(names)
-    return tuple(os.path.join(download_path, name) for name in names)
+    running_name, finished_name = tuple(download_path / name for name in names)
+    return DloadFile(running_name, finished_name, mode, is_zip)
 
 
 mem_id = 0

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_109.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -10,7 +9,7 @@ from flask import g
 from sqlalchemy import or_, text
 from sqlalchemy.sql.expression import null
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import SiteSource, contract_func, displaced_era, forecast_date
 from chellow.models import Contract, Era, Session, Site, SiteEra, Source, Supply
 from chellow.utils import c_months_u, hh_format, hh_range, req_int
@@ -34,10 +33,7 @@ def content(contract_id, end_year, end_month, months, user):
     f = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                "displaced.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            f = open_file("displaced.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             titles = [
                 "Site Code",
@@ -268,7 +264,6 @@ def content(contract_id, end_year, end_month, months, user):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_111.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -20,8 +19,8 @@ from werkzeug.exceptions import BadRequest
 
 from zish import ZishLocationException, loads
 
-import chellow.dloads
-from chellow.e.computer import contract_func
+from chellow.dloads import open_file
+from chellow.e.computer import SupplySource, contract_func
 from chellow.models import (
     Batch,
     Bill,
@@ -109,10 +108,9 @@ def content(
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(
-                f"bill_check_{fname_additional}.csv", user
+            tmp_file = open_file(
+                f"bill_check_{fname_additional}.csv", user, mode="w", newline=""
             )
-            tmp_file = open(running_name, mode="w", newline="")
             writer = csv.writer(tmp_file, lineterminator="\n")
 
             report_run = ReportRun.get_by_id(sess, report_run_id)
@@ -245,7 +243,6 @@ def content(
     finally:
         if tmp_file is not None:
             tmp_file.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
@@ -558,7 +555,7 @@ def _process_supply(
                 )
                 data_source = data_sources[ds_key]
             except KeyError:
-                data_source = data_sources[ds_key] = chellow.e.computer.SupplySource(
+                data_source = data_sources[ds_key] = SupplySource(
                     sess,
                     chunk_start,
                     chunk_finish,
@@ -573,7 +570,7 @@ def _process_supply(
             if data_source.measurement_type == "hh":
                 metered_kwh += sum(h["msp-kwh"] for h in data_source.hh_data)
             else:
-                ds = chellow.e.computer.SupplySource(
+                ds = SupplySource(
                     sess,
                     chunk_start,
                     chunk_finish,

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_169.py

@@ -1,8 +1,6 @@
 import csv
-import os
 import threading
 import traceback
-import zipfile
 from io import StringIO
 
 from flask import g, request
@@ -10,7 +8,7 @@ from flask import g, request
 from sqlalchemy import or_, select
 from sqlalchemy.sql.expression import null
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import SupplySource, forecast_date
 from chellow.models import Channel, Era, HhDatum, RSession, Supply, User
 from chellow.utils import (
@@ -105,13 +103,12 @@ def content(
             fdate = forecast_date()
             user = User.get_by_id(sess, user_id)
 
-            running_name, finished_name = chellow.dloads.make_names(
-                "_".join(base_name) + (".zip" if is_zipped else ".csv"), user
-            )
             if is_zipped:
-                zf = zipfile.ZipFile(running_name, "w", zipfile.ZIP_DEFLATED)
+                zf = open_file(
+                    "_".join(base_name) + ".zip", user, mode="w", is_zip=True
+                )
             else:
-                tf = open(running_name, mode="w", newline="")
+                tf = open_file("_".join(base_name) + ".csv", user, mode="w", newline="")
                 tf.write(titles_csv)
 
             for supply in sess.execute(supplies).scalars():
@@ -226,8 +223,6 @@ def content(
             zf.close()
         else:
             tf.write(msg)
-    finally:
-        os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_187.py

@@ -1,7 +1,5 @@
-import os
 import threading
 import traceback
-import zipfile
 from datetime import datetime as Datetime
 
 from flask import g, request
@@ -10,7 +8,7 @@ from sqlalchemy import null, or_, select, text, true
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import Era, Session, Site, SiteEra, Supply, User
 from chellow.utils import (
     hh_format,
@@ -67,12 +65,10 @@ def content(start_date, finish_date, supply_id, mpan_cores, is_zipped, user_id):
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
 
-            running_name, finished_name = chellow.dloads.make_names(base_name, user)
-
             if is_zipped:
-                zf = zipfile.ZipFile(running_name, "w")
+                zf = open_file(base_name, user, mode="w", is_zip=True)
             else:
-                tmp_file = open(running_name, "w")
+                tmp_file = open_file(base_name, user, mode="w")
 
             caches = {}
             supplies = (
@@ -243,11 +239,9 @@ def content(start_date, finish_date, supply_id, mpan_cores, is_zipped, user_id):
     finally:
         if is_zipped:
             zf.close()
-            os.rename(running_name, finished_name)
         else:
             if tmp_file is not None:
                 tmp_file.close()
-                os.rename(running_name, finished_name)
 
 
 def do_post(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_219.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 
@@ -10,7 +9,7 @@ from sqlalchemy.orm import joinedload
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import (
     Batch,
     Bill,
@@ -31,10 +30,7 @@ def content(year, month, months, supply_id, user_id):
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
 
-            running_name, finished_name = chellow.dloads.make_names(
-                "register_reads.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            f = open_file("register_reads.csv", user, mode="w", newline="")
             w = csv.writer(f, lineterminator="\n")
             titles = (
                 "Duration Start",
@@ -178,7 +174,6 @@ def content(year, month, months, supply_id, user_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_247.py

@@ -1,4 +1,3 @@
-import os
 import sys
 import threading
 import traceback
@@ -16,7 +15,7 @@ from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import contract_func, forecast_date
 from chellow.e.scenario import make_calcs, make_site_deltas
 from chellow.models import (
@@ -747,11 +746,7 @@ def content(
             if is_bill_check:
                 base_name.append("bill_check")
 
-            running_name, finished_name = chellow.dloads.make_names(
-                "_".join(base_name) + ".ods", user
-            )
-
-            rf = open(running_name, "wb")
+            rf = open_file("_".join(base_name) + ".ods", user, mode="wb")
 
             for rate_script in scenario_props.get("rates", []):
                 contract_id = rate_script["contract_id"]
@@ -1024,8 +1019,7 @@ def content(
         site_rows.append(["Problem " + msg])
         if rf is None:
             msg = traceback.format_exc()
-            r_name, f_name = chellow.dloads.make_names("error.txt", None)
-            ef = open(r_name, "w")
+            ef = open_file("error.txt", None, mode="w")
             ef.write(msg + "\n")
             ef.close()
         else:
@@ -1042,7 +1036,6 @@ def content(
     finally:
         if rf is not None:
             rf.close()
-            os.rename(running_name, finished_name)
 
 
 def do_post(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_29.py

@@ -1,12 +1,11 @@
 import csv
-import os
 import sys
 import threading
 import traceback
 
 from flask import g
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import Session, Site, User
 from chellow.utils import c_months_u, csv_make_val, req_int, req_str, to_ct
 from chellow.views import chellow_redirect
@@ -22,10 +21,12 @@ def content(start_date, finish_date, site_id, typ, user_id):
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(
-                f'site_hh_data_{to_ct(start_date).strftime("%Y%m%d%H%M")}.csv', user
+            f = open_file(
+                f'site_hh_data_{to_ct(start_date).strftime("%Y%m%d%H%M")}.csv',
+                user,
+                mode="w",
+                newline="",
             )
-            f = open(running_name, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             titles = ["site_code", "type", "hh_start_clock_time", "total"]
             hr_titles = tuple(map(str, range(1, 51)))
@@ -61,7 +62,6 @@ def content(start_date, finish_date, site_id, typ, user_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_387.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -9,7 +8,7 @@ from flask import g
 from sqlalchemy import or_, select
 from sqlalchemy.sql.expression import null, true
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import SupplySource, forecast_date
 from chellow.models import Era, Session, Site, SiteEra, Supply
 from chellow.utils import (
@@ -31,10 +30,9 @@ def content(supply_id, start_date, finish_date, user):
 
             f_date = forecast_date()
 
-            running_name, finished_name = chellow.dloads.make_names(
-                f"supply_virtual_bills_hh_{supply_id}.csv", user
+            f = open_file(
+                f"supply_virtual_bills_hh_{supply_id}.csv", user, mode="w", newline=""
             )
-            f = open(running_name, mode="w", newline="")
             w = csv.writer(f, lineterminator="\n")
 
             mop_titles = []
@@ -155,7 +153,6 @@ def content(supply_id, start_date, finish_date, user):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_41.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -11,7 +10,7 @@ from sqlalchemy import or_
 from sqlalchemy.sql.expression import null, true
 
 import chellow.e.tnuos
-from chellow.dloads import make_names
+from chellow.dloads import open_file
 from chellow.e.computer import SupplySource, forecast_date
 from chellow.e.duos import duos_vb
 from chellow.models import Era, Pc, Session, Site, SiteEra, Source, Supply, User
@@ -54,8 +53,7 @@ def content(year, supply_id, user_id):
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = make_names("supplies_triad.csv", user)
-            f = open(running_name, mode="w", newline="")
+            f = open_file("supplies_triad.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
 
             march_start = to_utc(ct_datetime(year, 3, 1))
@@ -207,7 +205,6 @@ def content(year, supply_id, user_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_429.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -15,8 +14,8 @@ from sqlalchemy.orm import joinedload
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
 import chellow.gas.engine
+from chellow.dloads import open_file
 from chellow.models import GBatch, GBill, GContract, GEra, Session, Site, SiteGEra, User
 from chellow.utils import csv_make_val, hh_max, hh_min, req_date, req_int, to_utc
 from chellow.views import chellow_redirect
@@ -30,10 +29,7 @@ def content(g_batch_id, g_bill_id, g_contract_id, start_date, finish_date, user_
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
 
-            running_name, finished_name = chellow.dloads.make_names(
-                "g_bill_check.csv", user
-            )
-            tmp_file = open(running_name, "w")
+            tmp_file = open_file("g_bill_check.csv", user, mode="w")
             csv_writer = csv.writer(tmp_file)
             g_bills = (
                 select(GBill)
@@ -121,7 +117,6 @@ def content(g_batch_id, g_bill_id, g_contract_id, start_date, finish_date, user_
         tmp_file.write(f"Problem {msg}")
     finally:
         tmp_file.close()
-        os.rename(running_name, finished_name)
 
 
 def _process_g_bill_ids(

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_59.py

@@ -1,4 +1,3 @@
-import os
 import sys
 import threading
 import traceback
@@ -15,7 +14,7 @@ from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.e.computer import contract_func, forecast_date
 from chellow.e.scenario import make_calcs, make_site_deltas
 from chellow.models import (
@@ -786,11 +785,7 @@ def content(
                 )
 
             user = User.get_by_id(rsess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(
-                "_".join(base_name) + ".ods", user
-            )
-
-            rf = open(running_name, "wb")
+            rf = open_file("_".join(base_name) + ".ods", user, mode="wb")
 
             for rate_script in scenario_props.get("rates", []):
                 contract_id = rate_script["contract_id"]
@@ -1094,8 +1089,7 @@ def content(
         site_rows.append(["Problem " + msg])
         if rf is None:
             msg = traceback.format_exc()
-            r_name, f_name = chellow.dloads.make_names("error.txt", None)
-            ef = open(r_name, "w")
+            ef = open_file("error.txt", None, mode="w")
             ef.write(msg + "\n")
             ef.close()
         else:
@@ -1113,7 +1107,6 @@ def content(
     finally:
         if rf is not None:
             rf.close()
-            os.rename(running_name, finished_name)
 
 
 def do_post(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_batches.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 
@@ -7,7 +6,7 @@ from flask import g
 
 from sqlalchemy.sql import func, select
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import (
     Batch,
     Bill,
@@ -25,8 +24,7 @@ def content(user_id):
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = chellow.dloads.make_names("batches.csv", user)
-            f = open(running_name, mode="w", newline="")
+            f = open_file("batches.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
 
             titles = (
@@ -140,7 +138,6 @@ def content(user_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):

{chellow-1706691842.0.0 → chellow-1707137154.0.0}/chellow/reports/report_bills.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import threading
 import traceback
 from collections import defaultdict
@@ -9,7 +8,7 @@ from flask import g
 from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 
-import chellow.dloads
+from chellow.dloads import open_file
 from chellow.models import Batch, Bill, Session, User
 from chellow.utils import csv_make_val, req_int
 from chellow.views import chellow_redirect
@@ -94,10 +93,7 @@ def content(user_id, batch_id):
     try:
         with Session() as sess:
             user = User.get_by_id(sess, user_id)
-            running_name, finished_name = chellow.dloads.make_names(
-                f"bills_batch_{batch_id}.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            f = open_file(f"bills_batch_{batch_id}.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             _content(sess, writer, batch_id)
 
@@ -109,7 +105,6 @@ def content(user_id, batch_id):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):